2024-11-10 19:16:00,955 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 19:16:00,971 main DEBUG Took 0.013583 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 19:16:00,972 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 19:16:00,972 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 19:16:00,974 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 19:16:00,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:00,986 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 19:16:01,003 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,006 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,007 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,009 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,010 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,011 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,012 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,013 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,014 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,014 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,015 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,015 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,016 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,017 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:16:01,018 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,019 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 19:16:01,020 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:16:01,022 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 19:16:01,025 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 19:16:01,025 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 19:16:01,027 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 19:16:01,028 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 19:16:01,039 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 19:16:01,043 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 19:16:01,045 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 19:16:01,045 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 19:16:01,046 main DEBUG createAppenders(={Console}) 2024-11-10 19:16:01,047 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-10 19:16:01,047 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 19:16:01,047 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-10 19:16:01,048 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 19:16:01,049 main DEBUG OutputStream closed 2024-11-10 19:16:01,049 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 19:16:01,049 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 19:16:01,050 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-10 19:16:01,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 19:16:01,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 19:16:01,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 19:16:01,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 19:16:01,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 19:16:01,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 19:16:01,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 19:16:01,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 19:16:01,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 19:16:01,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 19:16:01,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 19:16:01,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 19:16:01,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 19:16:01,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 19:16:01,155 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 19:16:01,155 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 19:16:01,155 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 19:16:01,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 19:16:01,159 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 19:16:01,159 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-10 19:16:01,160 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 19:16:01,161 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-10T19:16:01,423 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba 2024-11-10 19:16:01,426 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 19:16:01,426 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T19:16:01,435 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-10T19:16:01,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T19:16:01,476 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41, deleteOnExit=true 2024-11-10T19:16:01,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T19:16:01,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/test.cache.data in system properties and HBase conf 2024-11-10T19:16:01,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T19:16:01,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/hadoop.log.dir in system properties and HBase conf 2024-11-10T19:16:01,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T19:16:01,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T19:16:01,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T19:16:01,582 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T19:16:01,680 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:16:01,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T19:16:01,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T19:16:01,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T19:16:01,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T19:16:01,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T19:16:01,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T19:16:01,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T19:16:01,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T19:16:01,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T19:16:01,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/nfs.dump.dir in system properties and HBase conf 2024-11-10T19:16:01,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/java.io.tmpdir in system properties and HBase conf 2024-11-10T19:16:01,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T19:16:01,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T19:16:01,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T19:16:02,959 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T19:16:03,062 INFO [Time-limited test {}] log.Log(170): Logging initialized @3268ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T19:16:03,187 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T19:16:03,308 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T19:16:03,352 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T19:16:03,352 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T19:16:03,354 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T19:16:03,373 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T19:16:03,378 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/hadoop.log.dir/,AVAILABLE} 2024-11-10T19:16:03,380 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T19:16:03,688 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/java.io.tmpdir/jetty-localhost-40929-hadoop-hdfs-3_4_1-tests_jar-_-any-13576516767798820896/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T19:16:03,699 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40929} 2024-11-10T19:16:03,700 INFO [Time-limited test {}] server.Server(415): Started @3907ms 2024-11-10T19:16:04,545 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T19:16:04,559 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T19:16:04,565 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T19:16:04,565 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T19:16:04,565 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T19:16:04,569 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/hadoop.log.dir/,AVAILABLE} 2024-11-10T19:16:04,571 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T19:16:04,703 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/java.io.tmpdir/jetty-localhost-34691-hadoop-hdfs-3_4_1-tests_jar-_-any-2474063699183999669/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T19:16:04,704 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34691} 2024-11-10T19:16:04,704 INFO [Time-limited test {}] server.Server(415): Started @4911ms 2024-11-10T19:16:04,761 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T19:16:05,816 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41/data/data1/current/BP-1526193424-172.17.0.3-1731266162412/current, will proceed with Du for space computation calculation, 2024-11-10T19:16:05,816 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41/data/data2/current/BP-1526193424-172.17.0.3-1731266162412/current, will proceed with Du for space computation calculation, 2024-11-10T19:16:05,852 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T19:16:05,894 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbc567314aac7ff99 with lease ID 0xa969a6d49ceffe61: Processing first storage report for DS-25784a4b-4789-411d-ba7d-5e03552f2f22 from datanode DatanodeRegistration(127.0.0.1:38057, datanodeUuid=fb2a8015-4173-44ad-935d-47d25a540c3c, infoPort=38893, infoSecurePort=0, ipcPort=44135, storageInfo=lv=-57;cid=testClusterID;nsid=213744199;c=1731266162412) 2024-11-10T19:16:05,895 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc567314aac7ff99 with lease ID 0xa969a6d49ceffe61: from storage DS-25784a4b-4789-411d-ba7d-5e03552f2f22 node DatanodeRegistration(127.0.0.1:38057, datanodeUuid=fb2a8015-4173-44ad-935d-47d25a540c3c, infoPort=38893, infoSecurePort=0, ipcPort=44135, storageInfo=lv=-57;cid=testClusterID;nsid=213744199;c=1731266162412), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T19:16:05,896 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbc567314aac7ff99 with lease ID 0xa969a6d49ceffe61: Processing first storage report for DS-1b4eaa93-7266-43a5-9282-626a01f460a1 from datanode DatanodeRegistration(127.0.0.1:38057, datanodeUuid=fb2a8015-4173-44ad-935d-47d25a540c3c, infoPort=38893, infoSecurePort=0, ipcPort=44135, storageInfo=lv=-57;cid=testClusterID;nsid=213744199;c=1731266162412) 2024-11-10T19:16:05,896 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc567314aac7ff99 with lease ID 0xa969a6d49ceffe61: from storage DS-1b4eaa93-7266-43a5-9282-626a01f460a1 node DatanodeRegistration(127.0.0.1:38057, datanodeUuid=fb2a8015-4173-44ad-935d-47d25a540c3c, infoPort=38893, infoSecurePort=0, ipcPort=44135, storageInfo=lv=-57;cid=testClusterID;nsid=213744199;c=1731266162412), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T19:16:05,901 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba 2024-11-10T19:16:05,978 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41/zookeeper_0, clientPort=53441, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T19:16:05,986 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53441 2024-11-10T19:16:05,999 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T19:16:06,002 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T19:16:06,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741825_1001 (size=7) 2024-11-10T19:16:06,628 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0 with version=8 2024-11-10T19:16:06,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/hbase-staging 2024-11-10T19:16:06,732 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T19:16:07,039 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bdc25462c9ef:0 server-side Connection retries=45 2024-11-10T19:16:07,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:16:07,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T19:16:07,055 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T19:16:07,055 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:16:07,055 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T19:16:07,201 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T19:16:07,257 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T19:16:07,266 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T19:16:07,270 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T19:16:07,294 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8253 (auto-detected) 2024-11-10T19:16:07,295 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-10T19:16:07,313 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46761 2024-11-10T19:16:07,337 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46761 connecting to ZooKeeper ensemble=127.0.0.1:53441 2024-11-10T19:16:07,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:467610x0, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T19:16:07,461 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46761-0x101262d38720000 connected 2024-11-10T19:16:07,596 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T19:16:07,598 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T19:16:07,609 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:16:07,614 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0, hbase.cluster.distributed=false 2024-11-10T19:16:07,640 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T19:16:07,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46761 2024-11-10T19:16:07,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46761 2024-11-10T19:16:07,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46761 2024-11-10T19:16:07,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46761 2024-11-10T19:16:07,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46761 2024-11-10T19:16:07,762 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bdc25462c9ef:0 server-side Connection retries=45 2024-11-10T19:16:07,764 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:16:07,764 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T19:16:07,764 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T19:16:07,765 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:16:07,765 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T19:16:07,767 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T19:16:07,770 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T19:16:07,771 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34487 2024-11-10T19:16:07,773 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34487 connecting to ZooKeeper ensemble=127.0.0.1:53441 2024-11-10T19:16:07,774 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T19:16:07,777 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T19:16:07,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344870x0, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T19:16:07,795 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:344870x0, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:16:07,795 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34487-0x101262d38720001 connected 2024-11-10T19:16:07,799 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T19:16:07,806 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T19:16:07,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T19:16:07,814 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T19:16:07,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34487 2024-11-10T19:16:07,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34487 2024-11-10T19:16:07,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34487 2024-11-10T19:16:07,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34487 2024-11-10T19:16:07,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34487 2024-11-10T19:16:07,839 DEBUG [M:0;bdc25462c9ef:46761 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bdc25462c9ef:46761 2024-11-10T19:16:07,840 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:07,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:16:07,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:16:07,860 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:07,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T19:16:07,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:07,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:07,892 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T19:16:07,894 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bdc25462c9ef,46761,1731266166843 from backup master directory 2024-11-10T19:16:07,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:16:07,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:07,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:16:07,911 WARN [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T19:16:07,911 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:07,914 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T19:16:07,916 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T19:16:07,985 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/hbase.id] with ID: d2805b0d-199f-46f8-9715-ae1ce58e3026 2024-11-10T19:16:07,985 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/.tmp/hbase.id 2024-11-10T19:16:07,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741826_1002 (size=42) 2024-11-10T19:16:08,398 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/.tmp/hbase.id]:[hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/hbase.id] 2024-11-10T19:16:08,456 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T19:16:08,462 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T19:16:08,482 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-10T19:16:08,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:08,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:08,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741827_1003 (size=196) 2024-11-10T19:16:09,008 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:16:09,011 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T19:16:09,024 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:09,028 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:16:09,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741828_1004 (size=1189) 2024-11-10T19:16:09,480 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store 2024-11-10T19:16:09,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741829_1005 (size=34) 2024-11-10T19:16:09,909 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T19:16:09,914 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:16:09,915 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T19:16:09,915 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:16:09,915 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:16:09,917 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T19:16:09,917 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:16:09,917 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:16:09,918 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731266169915Disabling compacts and flushes for region at 1731266169915Disabling writes for close at 1731266169917 (+2 ms)Writing region close event to WAL at 1731266169917Closed at 1731266169917 2024-11-10T19:16:09,920 WARN [master/bdc25462c9ef:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/.initializing 2024-11-10T19:16:09,921 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/WALs/bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:09,929 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T19:16:09,943 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bdc25462c9ef%2C46761%2C1731266166843, suffix=, logDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/WALs/bdc25462c9ef,46761,1731266166843, archiveDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/oldWALs, maxLogs=10 2024-11-10T19:16:09,964 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/WALs/bdc25462c9ef,46761,1731266166843/bdc25462c9ef%2C46761%2C1731266166843.1731266169947, exclude list is [], retry=0 2024-11-10T19:16:09,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38057,DS-25784a4b-4789-411d-ba7d-5e03552f2f22,DISK] 2024-11-10T19:16:09,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T19:16:10,018 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/WALs/bdc25462c9ef,46761,1731266166843/bdc25462c9ef%2C46761%2C1731266166843.1731266169947 2024-11-10T19:16:10,020 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38893:38893)] 2024-11-10T19:16:10,020 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:16:10,021 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:16:10,025 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,026 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,069 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,095 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T19:16:10,098 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:10,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:16:10,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T19:16:10,105 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:10,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:10,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T19:16:10,110 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:10,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:10,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T19:16:10,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:10,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:10,116 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,119 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,121 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,126 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,126 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,130 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T19:16:10,133 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:16:10,138 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:16:10,139 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61680857, jitterRate=-0.08088360726833344}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T19:16:10,145 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731266170040Initializing all the Stores at 1731266170042 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731266170043 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266170044 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266170044Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266170045 (+1 ms)Cleaning up temporary data from old regions at 1731266170126 (+81 ms)Region opened successfully at 1731266170145 (+19 ms) 2024-11-10T19:16:10,146 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T19:16:10,180 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca190e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bdc25462c9ef/172.17.0.3:0 2024-11-10T19:16:10,211 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T19:16:10,220 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T19:16:10,221 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T19:16:10,223 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T19:16:10,224 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T19:16:10,228 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-10T19:16:10,228 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T19:16:10,253 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T19:16:10,263 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T19:16:10,273 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T19:16:10,275 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T19:16:10,277 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T19:16:10,288 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T19:16:10,291 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T19:16:10,297 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T19:16:10,309 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T19:16:10,311 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T19:16:10,320 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T19:16:10,342 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T19:16:10,399 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T19:16:10,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T19:16:10,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T19:16:10,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:10,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:10,418 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bdc25462c9ef,46761,1731266166843, sessionid=0x101262d38720000, setting cluster-up flag (Was=false) 2024-11-10T19:16:10,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:10,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:10,484 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T19:16:10,487 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:10,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:10,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:10,541 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T19:16:10,543 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:10,550 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T19:16:10,614 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T19:16:10,624 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T19:16:10,629 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(746): ClusterId : d2805b0d-199f-46f8-9715-ae1ce58e3026 2024-11-10T19:16:10,630 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T19:16:10,631 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T19:16:10,634 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bdc25462c9ef,46761,1731266166843 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T19:16:10,644 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T19:16:10,644 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T19:16:10,644 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bdc25462c9ef:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:16:10,645 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bdc25462c9ef:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:16:10,645 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bdc25462c9ef:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:16:10,645 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bdc25462c9ef:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:16:10,645 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bdc25462c9ef:0, corePoolSize=10, maxPoolSize=10 2024-11-10T19:16:10,645 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,645 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bdc25462c9ef:0, corePoolSize=2, maxPoolSize=2 2024-11-10T19:16:10,645 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,648 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731266200648 2024-11-10T19:16:10,650 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T19:16:10,651 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T19:16:10,653 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T19:16:10,653 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T19:16:10,654 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T19:16:10,654 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T19:16:10,655 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T19:16:10,655 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T19:16:10,655 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,658 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T19:16:10,659 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T19:16:10,659 DEBUG [RS:0;bdc25462c9ef:34487 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c283a8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bdc25462c9ef/172.17.0.3:0 2024-11-10T19:16:10,661 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T19:16:10,661 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T19:16:10,661 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:10,661 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T19:16:10,663 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T19:16:10,663 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T19:16:10,665 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bdc25462c9ef:0:becomeActiveMaster-HFileCleaner.large.0-1731266170664,5,FailOnTimeoutGroup] 2024-11-10T19:16:10,666 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bdc25462c9ef:0:becomeActiveMaster-HFileCleaner.small.0-1731266170665,5,FailOnTimeoutGroup] 2024-11-10T19:16:10,666 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,666 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T19:16:10,668 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,668 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741831_1007 (size=1321) 2024-11-10T19:16:10,676 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bdc25462c9ef:34487 2024-11-10T19:16:10,679 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T19:16:10,679 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T19:16:10,679 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T19:16:10,682 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(2659): reportForDuty to master=bdc25462c9ef,46761,1731266166843 with port=34487, startcode=1731266167729 2024-11-10T19:16:10,692 DEBUG [RS:0;bdc25462c9ef:34487 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T19:16:10,730 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32953, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T19:16:10,739 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46761 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:10,742 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46761 {}] master.ServerManager(517): Registering regionserver=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:10,758 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0 2024-11-10T19:16:10,758 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34609 2024-11-10T19:16:10,758 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T19:16:10,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:16:10,773 DEBUG [RS:0;bdc25462c9ef:34487 {}] zookeeper.ZKUtil(111): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:10,774 WARN [RS:0;bdc25462c9ef:34487 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T19:16:10,775 INFO [RS:0;bdc25462c9ef:34487 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:16:10,775 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/WALs/bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:10,777 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bdc25462c9ef,34487,1731266167729] 2024-11-10T19:16:10,806 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T19:16:10,827 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T19:16:10,833 INFO [RS:0;bdc25462c9ef:34487 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T19:16:10,833 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,837 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T19:16:10,845 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T19:16:10,847 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,848 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,848 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,848 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,848 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,848 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,848 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bdc25462c9ef:0, corePoolSize=2, maxPoolSize=2 2024-11-10T19:16:10,849 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,849 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,849 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,849 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,849 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,850 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bdc25462c9ef:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:16:10,850 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bdc25462c9ef:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:16:10,850 DEBUG [RS:0;bdc25462c9ef:34487 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:16:10,851 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,851 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,851 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,852 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,852 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,852 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=bdc25462c9ef,34487,1731266167729-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T19:16:10,873 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T19:16:10,875 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=bdc25462c9ef,34487,1731266167729-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,876 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,876 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.Replication(171): bdc25462c9ef,34487,1731266167729 started 2024-11-10T19:16:10,900 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:10,901 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1482): Serving as bdc25462c9ef,34487,1731266167729, RpcServer on bdc25462c9ef/172.17.0.3:34487, sessionid=0x101262d38720001 2024-11-10T19:16:10,901 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T19:16:10,902 DEBUG [RS:0;bdc25462c9ef:34487 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:10,902 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bdc25462c9ef,34487,1731266167729' 2024-11-10T19:16:10,902 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T19:16:10,903 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T19:16:10,904 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T19:16:10,904 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T19:16:10,904 DEBUG [RS:0;bdc25462c9ef:34487 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:10,904 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bdc25462c9ef,34487,1731266167729' 2024-11-10T19:16:10,904 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T19:16:10,905 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T19:16:10,906 DEBUG [RS:0;bdc25462c9ef:34487 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T19:16:10,906 INFO [RS:0;bdc25462c9ef:34487 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T19:16:10,906 INFO [RS:0;bdc25462c9ef:34487 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T19:16:11,014 INFO [RS:0;bdc25462c9ef:34487 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T19:16:11,018 INFO [RS:0;bdc25462c9ef:34487 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bdc25462c9ef%2C34487%2C1731266167729, suffix=, logDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/WALs/bdc25462c9ef,34487,1731266167729, archiveDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/oldWALs, maxLogs=32 2024-11-10T19:16:11,034 DEBUG [RS:0;bdc25462c9ef:34487 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/WALs/bdc25462c9ef,34487,1731266167729/bdc25462c9ef%2C34487%2C1731266167729.1731266171021, exclude list is [], retry=0 2024-11-10T19:16:11,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38057,DS-25784a4b-4789-411d-ba7d-5e03552f2f22,DISK] 2024-11-10T19:16:11,042 INFO [RS:0;bdc25462c9ef:34487 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/WALs/bdc25462c9ef,34487,1731266167729/bdc25462c9ef%2C34487%2C1731266167729.1731266171021 2024-11-10T19:16:11,043 DEBUG [RS:0;bdc25462c9ef:34487 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38893:38893)] 2024-11-10T19:16:11,076 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T19:16:11,077 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0 2024-11-10T19:16:11,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741833_1009 (size=32) 2024-11-10T19:16:11,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:16:11,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T19:16:11,499 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T19:16:11,500 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:11,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:16:11,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T19:16:11,505 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T19:16:11,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:11,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:16:11,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T19:16:11,510 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T19:16:11,511 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:11,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:16:11,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T19:16:11,515 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T19:16:11,515 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:11,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:16:11,517 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T19:16:11,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740 2024-11-10T19:16:11,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740 2024-11-10T19:16:11,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T19:16:11,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T19:16:11,525 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:16:11,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T19:16:11,537 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:16:11,538 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65036528, jitterRate=-0.030880212783813477}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:16:11,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731266171493Initializing all the Stores at 1731266171495 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731266171495Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731266171496 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266171496Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731266171496Cleaning up temporary data from old regions at 1731266171524 (+28 ms)Region opened successfully at 1731266171540 (+16 ms) 2024-11-10T19:16:11,541 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T19:16:11,541 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T19:16:11,541 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T19:16:11,541 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T19:16:11,541 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T19:16:11,542 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T19:16:11,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731266171540Disabling compacts and flushes for region at 1731266171540Disabling writes for close at 1731266171541 (+1 ms)Writing region close event to WAL at 1731266171542 (+1 ms)Closed at 1731266171542 2024-11-10T19:16:11,546 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T19:16:11,546 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T19:16:11,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T19:16:11,564 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T19:16:11,567 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T19:16:11,721 DEBUG [bdc25462c9ef:46761 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T19:16:11,739 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:11,745 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bdc25462c9ef,34487,1731266167729, state=OPENING 2024-11-10T19:16:11,888 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T19:16:11,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:11,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:16:11,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:16:11,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:16:11,901 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T19:16:11,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:16:12,077 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T19:16:12,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32855, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T19:16:12,091 INFO [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T19:16:12,091 INFO [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:16:12,092 INFO [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T19:16:12,095 INFO [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bdc25462c9ef%2C34487%2C1731266167729.meta, suffix=.meta, logDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/WALs/bdc25462c9ef,34487,1731266167729, archiveDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/oldWALs, maxLogs=32 2024-11-10T19:16:12,109 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/WALs/bdc25462c9ef,34487,1731266167729/bdc25462c9ef%2C34487%2C1731266167729.meta.1731266172097.meta, exclude list is [], retry=0 2024-11-10T19:16:12,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38057,DS-25784a4b-4789-411d-ba7d-5e03552f2f22,DISK] 2024-11-10T19:16:12,116 INFO [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/WALs/bdc25462c9ef,34487,1731266167729/bdc25462c9ef%2C34487%2C1731266167729.meta.1731266172097.meta 2024-11-10T19:16:12,116 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38893:38893)] 2024-11-10T19:16:12,116 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:16:12,118 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T19:16:12,120 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T19:16:12,124 INFO [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T19:16:12,127 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T19:16:12,128 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:16:12,128 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T19:16:12,128 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T19:16:12,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T19:16:12,134 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T19:16:12,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:12,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:16:12,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T19:16:12,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T19:16:12,138 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:12,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:16:12,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T19:16:12,142 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T19:16:12,142 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:12,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:16:12,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T19:16:12,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T19:16:12,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:12,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:16:12,148 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T19:16:12,150 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740 2024-11-10T19:16:12,154 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740 2024-11-10T19:16:12,157 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T19:16:12,157 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T19:16:12,159 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:16:12,163 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T19:16:12,165 INFO [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62901347, jitterRate=-0.06269688904285431}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:16:12,165 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T19:16:12,167 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731266172129Writing region info on filesystem at 1731266172129Initializing all the Stores at 1731266172131 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731266172131Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731266172131Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266172131Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731266172131Cleaning up temporary data from old regions at 1731266172157 (+26 ms)Running coprocessor post-open hooks at 1731266172165 (+8 ms)Region opened successfully at 1731266172167 (+2 ms) 2024-11-10T19:16:12,175 INFO [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731266172067 2024-11-10T19:16:12,186 DEBUG [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T19:16:12,187 INFO [RS_OPEN_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T19:16:12,188 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:12,191 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bdc25462c9ef,34487,1731266167729, state=OPEN 2024-11-10T19:16:12,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T19:16:12,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T19:16:12,279 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:16:12,279 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:16:12,279 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:12,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T19:16:12,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bdc25462c9ef,34487,1731266167729 in 376 msec 2024-11-10T19:16:12,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T19:16:12,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 739 msec 2024-11-10T19:16:12,299 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T19:16:12,299 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T19:16:12,320 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:12,321 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:12,345 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:12,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54037, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:12,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7860 sec 2024-11-10T19:16:12,366 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731266172366, completionTime=-1 2024-11-10T19:16:12,369 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T19:16:12,369 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T19:16:12,394 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T19:16:12,395 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731266232395 2024-11-10T19:16:12,395 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731266292395 2024-11-10T19:16:12,395 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 26 msec 2024-11-10T19:16:12,397 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bdc25462c9ef,46761,1731266166843-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:12,397 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bdc25462c9ef,46761,1731266166843-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:12,397 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bdc25462c9ef,46761,1731266166843-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:12,399 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bdc25462c9ef:46761, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:12,399 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:12,400 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:12,405 DEBUG [master/bdc25462c9ef:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T19:16:12,427 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.515sec 2024-11-10T19:16:12,428 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T19:16:12,430 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T19:16:12,430 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T19:16:12,431 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T19:16:12,431 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T19:16:12,432 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bdc25462c9ef,46761,1731266166843-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T19:16:12,432 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bdc25462c9ef,46761,1731266166843-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T19:16:12,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:12,441 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T19:16:12,442 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T19:16:12,442 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T19:16:12,442 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T19:16:12,442 INFO [master/bdc25462c9ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bdc25462c9ef,46761,1731266166843-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:16:12,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:12,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:12,455 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:12,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:12,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:12,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4889feab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:12,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:12,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:12,464 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:12,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:12,467 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:12,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:12,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:12,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=593, ProcessCount=11, AvailableMemoryMB=3499 2024-11-10T19:16:12,524 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T19:16:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:12,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65d48f8e 2024-11-10T19:16:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T19:16:12,538 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T19:16:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T19:16:12,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:16:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:12,555 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T19:16:12,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T19:16:12,558 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:12,561 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T19:16:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:16:12,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741835_1011 (size=1061) 2024-11-10T19:16:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:16:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:16:12,979 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0 2024-11-10T19:16:12,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741836_1012 (size=53) 2024-11-10T19:16:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:16:13,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:16:13,394 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4f5e56bdcde85bbc88e3b9e5355a12b5, disabling compactions & flushes 2024-11-10T19:16:13,394 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:13,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:13,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. after waiting 0 ms 2024-11-10T19:16:13,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:13,396 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:13,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: Waiting for close lock at 1731266173394Disabling compacts and flushes for region at 1731266173394Disabling writes for close at 1731266173395 (+1 ms)Writing region close event to WAL at 1731266173395Closed at 1731266173395 2024-11-10T19:16:13,400 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T19:16:13,405 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731266173401"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731266173401"}]},"ts":"1731266173401"} 2024-11-10T19:16:13,409 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T19:16:13,411 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T19:16:13,413 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266173411"}]},"ts":"1731266173411"} 2024-11-10T19:16:13,418 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T19:16:13,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5, ASSIGN}] 2024-11-10T19:16:13,422 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5, ASSIGN 2024-11-10T19:16:13,425 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5, ASSIGN; state=OFFLINE, location=bdc25462c9ef,34487,1731266167729; forceNewPlan=false, retain=false 2024-11-10T19:16:13,578 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4f5e56bdcde85bbc88e3b9e5355a12b5, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:13,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5, ASSIGN because future has completed 2024-11-10T19:16:13,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:16:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:16:13,748 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:13,748 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:16:13,749 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,749 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:16:13,749 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,749 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,751 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,754 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:16:13,755 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f5e56bdcde85bbc88e3b9e5355a12b5 columnFamilyName A 2024-11-10T19:16:13,755 DEBUG [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:13,756 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] regionserver.HStore(327): Store=4f5e56bdcde85bbc88e3b9e5355a12b5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:13,756 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,759 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:16:13,759 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f5e56bdcde85bbc88e3b9e5355a12b5 columnFamilyName B 2024-11-10T19:16:13,759 DEBUG [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:13,760 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] regionserver.HStore(327): Store=4f5e56bdcde85bbc88e3b9e5355a12b5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:13,760 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,763 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:16:13,763 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f5e56bdcde85bbc88e3b9e5355a12b5 columnFamilyName C 2024-11-10T19:16:13,764 DEBUG [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:13,764 INFO [StoreOpener-4f5e56bdcde85bbc88e3b9e5355a12b5-1 {}] regionserver.HStore(327): Store=4f5e56bdcde85bbc88e3b9e5355a12b5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:13,765 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:13,765 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,767 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,768 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,770 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,770 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,771 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:16:13,774 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,777 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:16:13,778 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4f5e56bdcde85bbc88e3b9e5355a12b5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70225488, jitterRate=0.04644131660461426}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:16:13,778 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:13,779 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: Running coprocessor pre-open hook at 1731266173749Writing region info on filesystem at 1731266173749Initializing all the Stores at 1731266173751 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266173751Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266173751Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266173751Cleaning up temporary data from old regions at 1731266173770 (+19 ms)Running coprocessor post-open hooks at 1731266173778 (+8 ms)Region opened successfully at 1731266173779 (+1 ms) 2024-11-10T19:16:13,781 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., pid=6, masterSystemTime=1731266173740 2024-11-10T19:16:13,785 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:13,785 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:13,786 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4f5e56bdcde85bbc88e3b9e5355a12b5, regionState=OPEN, openSeqNum=2, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:13,788 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46761 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=bdc25462c9ef,34487,1731266167729, table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-10T19:16:13,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:16:13,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T19:16:13,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 in 208 msec 2024-11-10T19:16:13,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T19:16:13,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5, ASSIGN in 378 msec 2024-11-10T19:16:13,803 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T19:16:13,803 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266173803"}]},"ts":"1731266173803"} 2024-11-10T19:16:13,806 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T19:16:13,808 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T19:16:13,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2610 sec 2024-11-10T19:16:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:16:14,715 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:14,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6900c275, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,730 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a12729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,732 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,739 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T19:16:14,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:14,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@737a82e7 2024-11-10T19:16:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T19:16:14,745 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T19:16:14,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,751 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,755 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,759 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,763 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,767 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,771 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,779 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bffd9e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,782 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f8a4b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,785 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103656b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,789 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbf9ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,793 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19664801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,797 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf5031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,800 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e54299b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,804 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ae3f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bee5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:14,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:14,812 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ddb883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:14,815 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:14,842 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,842 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,842 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,843 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,844 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,844 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,844 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,844 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,848 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:16:14,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:16:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:16:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:16:14,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:16:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:16:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:16:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:16:14,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T19:16:14,887 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T19:16:14,893 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:14,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:14,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T19:16:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T19:16:15,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:15,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:15,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:15,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:15,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:15,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:15,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T19:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266235063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266235069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266235072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266235073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266235074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:15,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/c731462d6b6d47bf8d2d9798518a6bc7 is 50, key is test_row_0/A:col10/1731266174946/Put/seqid=0 2024-11-10T19:16:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741837_1013 (size=19021) 2024-11-10T19:16:15,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/c731462d6b6d47bf8d2d9798518a6bc7 2024-11-10T19:16:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T19:16:15,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266235243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266235244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266235244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266235245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266235245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/43a6320633b9488181a44979dca5bfae is 50, key is test_row_0/B:col10/1731266174946/Put/seqid=0 2024-11-10T19:16:15,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T19:16:15,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741838_1014 (size=12001) 2024-11-10T19:16:15,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T19:16:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266235463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266235464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266235465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266235465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266235465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T19:16:15,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T19:16:15,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:15,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/43a6320633b9488181a44979dca5bfae 2024-11-10T19:16:15,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T19:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:15,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/f9ae93e29fa44b1f909797b66121b124 is 50, key is test_row_0/C:col10/1731266174946/Put/seqid=0 2024-11-10T19:16:15,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266235784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266235783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266235785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741839_1015 (size=12001) 2024-11-10T19:16:15,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266235785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/f9ae93e29fa44b1f909797b66121b124 2024-11-10T19:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266235785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:15,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/c731462d6b6d47bf8d2d9798518a6bc7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c731462d6b6d47bf8d2d9798518a6bc7 2024-11-10T19:16:15,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c731462d6b6d47bf8d2d9798518a6bc7, entries=300, sequenceid=17, filesize=18.6 K 2024-11-10T19:16:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/43a6320633b9488181a44979dca5bfae as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43a6320633b9488181a44979dca5bfae 2024-11-10T19:16:15,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43a6320633b9488181a44979dca5bfae, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T19:16:15,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/f9ae93e29fa44b1f909797b66121b124 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/f9ae93e29fa44b1f909797b66121b124 2024-11-10T19:16:15,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/f9ae93e29fa44b1f909797b66121b124, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T19:16:15,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 908ms, sequenceid=17, compaction requested=false 2024-11-10T19:16:15,897 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T19:16:15,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:15,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T19:16:15,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:15,913 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:15,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/5b5baf27efe24231a8417781b236d422 is 50, key is test_row_0/A:col10/1731266175061/Put/seqid=0 2024-11-10T19:16:15,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741840_1016 (size=12001) 2024-11-10T19:16:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T19:16:16,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:16,355 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/5b5baf27efe24231a8417781b236d422 2024-11-10T19:16:16,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266236356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266236371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266236371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266236379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266236375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/77386700e04a4c00bd34eec8aef0924c is 50, key is test_row_0/B:col10/1731266175061/Put/seqid=0 2024-11-10T19:16:16,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741841_1017 (size=12001) 2024-11-10T19:16:16,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266236482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266236497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266236499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266236503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266236504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266236692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266236712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266236713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266236722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266236722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:16,798 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T19:16:16,840 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/77386700e04a4c00bd34eec8aef0924c 2024-11-10T19:16:16,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/40f570aa005a4ba595c8e22dbbaddb48 is 50, key is test_row_0/C:col10/1731266175061/Put/seqid=0 2024-11-10T19:16:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741842_1018 (size=12001) 2024-11-10T19:16:16,950 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/40f570aa005a4ba595c8e22dbbaddb48 2024-11-10T19:16:16,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/5b5baf27efe24231a8417781b236d422 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5b5baf27efe24231a8417781b236d422 2024-11-10T19:16:16,983 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5b5baf27efe24231a8417781b236d422, entries=150, sequenceid=34, filesize=11.7 K 2024-11-10T19:16:16,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/77386700e04a4c00bd34eec8aef0924c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/77386700e04a4c00bd34eec8aef0924c 2024-11-10T19:16:17,003 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/77386700e04a4c00bd34eec8aef0924c, entries=150, sequenceid=34, filesize=11.7 K 2024-11-10T19:16:17,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/40f570aa005a4ba595c8e22dbbaddb48 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/40f570aa005a4ba595c8e22dbbaddb48 2024-11-10T19:16:17,021 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/40f570aa005a4ba595c8e22dbbaddb48, entries=150, sequenceid=34, filesize=11.7 K 2024-11-10T19:16:17,023 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 1110ms, sequenceid=34, compaction requested=false 2024-11-10T19:16:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T19:16:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T19:16:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T19:16:17,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T19:16:17,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1310 sec 2024-11-10T19:16:17,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.1650 sec 2024-11-10T19:16:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:17,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:17,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:17,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:17,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:17,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:17,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:17,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:17,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a47d329c598f433c859c7f1f57e0a4c6 is 50, key is test_row_0/A:col10/1731266177043/Put/seqid=0 2024-11-10T19:16:17,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741843_1019 (size=12001) 2024-11-10T19:16:17,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a47d329c598f433c859c7f1f57e0a4c6 2024-11-10T19:16:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266237098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266237102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266237103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266237107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266237115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/bb81878f2e8146e282e18493455a13c4 is 50, key is test_row_0/B:col10/1731266177043/Put/seqid=0 2024-11-10T19:16:17,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741844_1020 (size=12001) 2024-11-10T19:16:17,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/bb81878f2e8146e282e18493455a13c4 2024-11-10T19:16:17,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/1c9f3c1963e84fdfab2d6c2e57deaf9d is 50, key is test_row_0/C:col10/1731266177043/Put/seqid=0 2024-11-10T19:16:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266237242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266237243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266237243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266237252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266237252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741845_1021 (size=12001) 2024-11-10T19:16:17,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/1c9f3c1963e84fdfab2d6c2e57deaf9d 2024-11-10T19:16:17,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a47d329c598f433c859c7f1f57e0a4c6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a47d329c598f433c859c7f1f57e0a4c6 2024-11-10T19:16:17,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a47d329c598f433c859c7f1f57e0a4c6, entries=150, sequenceid=52, filesize=11.7 K 2024-11-10T19:16:17,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/bb81878f2e8146e282e18493455a13c4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/bb81878f2e8146e282e18493455a13c4 2024-11-10T19:16:17,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/bb81878f2e8146e282e18493455a13c4, entries=150, sequenceid=52, filesize=11.7 K 2024-11-10T19:16:17,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/1c9f3c1963e84fdfab2d6c2e57deaf9d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/1c9f3c1963e84fdfab2d6c2e57deaf9d 2024-11-10T19:16:17,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/1c9f3c1963e84fdfab2d6c2e57deaf9d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-10T19:16:17,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 313ms, sequenceid=52, compaction requested=true 2024-11-10T19:16:17,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:17,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:17,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:17,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:17,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:17,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:17,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:17,385 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:17,385 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:17,390 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:17,398 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:17,398 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:17,399 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c731462d6b6d47bf8d2d9798518a6bc7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5b5baf27efe24231a8417781b236d422, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a47d329c598f433c859c7f1f57e0a4c6] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=42.0 K 2024-11-10T19:16:17,402 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:17,402 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:17,402 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c731462d6b6d47bf8d2d9798518a6bc7, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731266174927 2024-11-10T19:16:17,402 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:17,402 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43a6320633b9488181a44979dca5bfae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/77386700e04a4c00bd34eec8aef0924c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/bb81878f2e8146e282e18493455a13c4] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=35.2 K 2024-11-10T19:16:17,404 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a6320633b9488181a44979dca5bfae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731266174946 2024-11-10T19:16:17,404 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b5baf27efe24231a8417781b236d422, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731266175061 2024-11-10T19:16:17,405 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 77386700e04a4c00bd34eec8aef0924c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731266175061 2024-11-10T19:16:17,405 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting a47d329c598f433c859c7f1f57e0a4c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731266176363 2024-11-10T19:16:17,406 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting bb81878f2e8146e282e18493455a13c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731266176363 2024-11-10T19:16:17,470 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:17,470 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#9 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:17,471 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/7513d82d7b5342adaf2d64e7cbd131c5 is 50, key is test_row_0/A:col10/1731266177043/Put/seqid=0 2024-11-10T19:16:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:17,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:17,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:17,473 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/dddb011cb6f24bbb8cb6944c0608bdf4 is 50, key is test_row_0/B:col10/1731266177043/Put/seqid=0 2024-11-10T19:16:17,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:17,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:17,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:17,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:17,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:17,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/2ca277d8bce94fbeb6d07b7df52d4f71 is 50, key is test_row_0/A:col10/1731266177101/Put/seqid=0 2024-11-10T19:16:17,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741846_1022 (size=12104) 2024-11-10T19:16:17,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741847_1023 (size=12104) 2024-11-10T19:16:17,512 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/7513d82d7b5342adaf2d64e7cbd131c5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/7513d82d7b5342adaf2d64e7cbd131c5 2024-11-10T19:16:17,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741848_1024 (size=14341) 2024-11-10T19:16:17,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/2ca277d8bce94fbeb6d07b7df52d4f71 2024-11-10T19:16:17,543 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 7513d82d7b5342adaf2d64e7cbd131c5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:17,543 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:17,546 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=13, startTime=1731266177358; duration=0sec 2024-11-10T19:16:17,547 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:17,547 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:17,548 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:17,559 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:17,559 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:17,559 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:17,559 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/f9ae93e29fa44b1f909797b66121b124, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/40f570aa005a4ba595c8e22dbbaddb48, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/1c9f3c1963e84fdfab2d6c2e57deaf9d] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=35.2 K 2024-11-10T19:16:17,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266237537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266237543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,563 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9ae93e29fa44b1f909797b66121b124, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731266174946 2024-11-10T19:16:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266237547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266237554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266237537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,565 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40f570aa005a4ba595c8e22dbbaddb48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731266175061 2024-11-10T19:16:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,567 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c9f3c1963e84fdfab2d6c2e57deaf9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731266176363 2024-11-10T19:16:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/1f1c95df63184aeeba84e56454870eed is 50, key is test_row_0/B:col10/1731266177101/Put/seqid=0 2024-11-10T19:16:17,597 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:17,598 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/e23dc7ddc9ac44739d7f240de5dd35bd is 50, key is test_row_0/C:col10/1731266177043/Put/seqid=0 2024-11-10T19:16:17,643 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T19:16:17,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741849_1025 (size=12001) 2024-11-10T19:16:17,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741850_1026 (size=12104) 2024-11-10T19:16:17,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266237671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266237672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266237673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266237673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266237673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,687 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/e23dc7ddc9ac44739d7f240de5dd35bd as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e23dc7ddc9ac44739d7f240de5dd35bd 2024-11-10T19:16:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,702 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into e23dc7ddc9ac44739d7f240de5dd35bd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:17,702 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:17,703 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=13, startTime=1731266177384; duration=0sec 2024-11-10T19:16:17,703 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:17,703 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:17,787 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T19:16:17,788 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T19:16:17,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T19:16:17,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T19:16:17,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T19:16:17,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T19:16:17,793 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T19:16:17,793 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T19:16:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266237883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266237893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266237893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266237893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266237893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:17,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:17,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:17,940 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/dddb011cb6f24bbb8cb6944c0608bdf4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/dddb011cb6f24bbb8cb6944c0608bdf4 2024-11-10T19:16:17,962 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into dddb011cb6f24bbb8cb6944c0608bdf4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:17,962 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:17,962 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=13, startTime=1731266177384; duration=0sec 2024-11-10T19:16:17,963 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:17,963 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:18,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/1f1c95df63184aeeba84e56454870eed 2024-11-10T19:16:18,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/487c373511ac43c4978a74af134571c5 is 50, key is test_row_0/C:col10/1731266177101/Put/seqid=0 2024-11-10T19:16:18,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741851_1027 (size=12001) 2024-11-10T19:16:18,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/487c373511ac43c4978a74af134571c5 2024-11-10T19:16:18,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/2ca277d8bce94fbeb6d07b7df52d4f71 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/2ca277d8bce94fbeb6d07b7df52d4f71 2024-11-10T19:16:18,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/2ca277d8bce94fbeb6d07b7df52d4f71, entries=200, sequenceid=68, filesize=14.0 K 2024-11-10T19:16:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/1f1c95df63184aeeba84e56454870eed as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1f1c95df63184aeeba84e56454870eed 2024-11-10T19:16:18,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1f1c95df63184aeeba84e56454870eed, entries=150, sequenceid=68, filesize=11.7 K 2024-11-10T19:16:18,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/487c373511ac43c4978a74af134571c5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/487c373511ac43c4978a74af134571c5 2024-11-10T19:16:18,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266238204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266238205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266238212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266238222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266238232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/487c373511ac43c4978a74af134571c5, entries=150, sequenceid=68, filesize=11.7 K 2024-11-10T19:16:18,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 780ms, sequenceid=68, compaction requested=false 2024-11-10T19:16:18,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:18,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:18,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:18,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:18,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:18,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:18,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:18,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:18,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a0db9049a52f4e9c968ff00cc115c7e5 is 50, key is test_row_0/A:col10/1731266178734/Put/seqid=0 2024-11-10T19:16:18,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266238886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741852_1028 (size=12001) 2024-11-10T19:16:18,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266238887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a0db9049a52f4e9c968ff00cc115c7e5 2024-11-10T19:16:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266238886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266238910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266238913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:18,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/2edbdbe84b3d4f309dedfa1a446602cc is 50, key is test_row_0/B:col10/1731266178734/Put/seqid=0 2024-11-10T19:16:19,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741853_1029 (size=12001) 2024-11-10T19:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266239011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266239022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266239022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266239032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T19:16:19,053 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:19,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T19:16:19,067 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:19,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266239055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,069 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T19:16:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T19:16:19,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266239233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266239233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266239233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266239252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266239271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:19,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:19,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T19:16:19,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/2edbdbe84b3d4f309dedfa1a446602cc 2024-11-10T19:16:19,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/16c1dec03cdf474c8fd40f1cf8f21b10 is 50, key is test_row_0/C:col10/1731266178734/Put/seqid=0 2024-11-10T19:16:19,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741854_1030 (size=12001) 2024-11-10T19:16:19,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/16c1dec03cdf474c8fd40f1cf8f21b10 2024-11-10T19:16:19,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:19,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a0db9049a52f4e9c968ff00cc115c7e5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a0db9049a52f4e9c968ff00cc115c7e5 2024-11-10T19:16:19,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:19,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266239552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266239552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266239552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a0db9049a52f4e9c968ff00cc115c7e5, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T19:16:19,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/2edbdbe84b3d4f309dedfa1a446602cc as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2edbdbe84b3d4f309dedfa1a446602cc 2024-11-10T19:16:19,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266239572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2edbdbe84b3d4f309dedfa1a446602cc, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T19:16:19,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/16c1dec03cdf474c8fd40f1cf8f21b10 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/16c1dec03cdf474c8fd40f1cf8f21b10 2024-11-10T19:16:19,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/16c1dec03cdf474c8fd40f1cf8f21b10, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T19:16:19,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 864ms, sequenceid=89, compaction requested=true 2024-11-10T19:16:19,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:19,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:19,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:19,605 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:19,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:19,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:19,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:19,605 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:19,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:19,612 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:19,612 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:19,612 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,613 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/dddb011cb6f24bbb8cb6944c0608bdf4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1f1c95df63184aeeba84e56454870eed, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2edbdbe84b3d4f309dedfa1a446602cc] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=35.3 K 2024-11-10T19:16:19,613 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:19,613 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:19,613 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,614 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/7513d82d7b5342adaf2d64e7cbd131c5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/2ca277d8bce94fbeb6d07b7df52d4f71, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a0db9049a52f4e9c968ff00cc115c7e5] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=37.5 K 2024-11-10T19:16:19,615 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting dddb011cb6f24bbb8cb6944c0608bdf4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731266176363 2024-11-10T19:16:19,615 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 7513d82d7b5342adaf2d64e7cbd131c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731266176363 2024-11-10T19:16:19,615 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f1c95df63184aeeba84e56454870eed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731266177096 2024-11-10T19:16:19,616 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ca277d8bce94fbeb6d07b7df52d4f71, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731266177096 2024-11-10T19:16:19,617 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting a0db9049a52f4e9c968ff00cc115c7e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731266177519 2024-11-10T19:16:19,617 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2edbdbe84b3d4f309dedfa1a446602cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731266177519 2024-11-10T19:16:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:19,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:19,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:19,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:19,659 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:19,660 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/46c72a6b2c3f49f7bc7c448fb2013de6 is 50, key is test_row_0/B:col10/1731266178734/Put/seqid=0 2024-11-10T19:16:19,680 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:19,681 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/6b209552aae9424bb1611415b7a24849 is 50, key is test_row_0/A:col10/1731266178734/Put/seqid=0 2024-11-10T19:16:19,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/8d4edd23a6eb4f5a95b9707b599db202 is 50, key is test_row_0/A:col10/1731266179613/Put/seqid=0 2024-11-10T19:16:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T19:16:19,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741855_1031 (size=12207) 2024-11-10T19:16:19,756 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/46c72a6b2c3f49f7bc7c448fb2013de6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/46c72a6b2c3f49f7bc7c448fb2013de6 2024-11-10T19:16:19,775 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 46c72a6b2c3f49f7bc7c448fb2013de6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:19,776 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:19,776 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=13, startTime=1731266179605; duration=0sec 2024-11-10T19:16:19,776 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:19,777 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:19,777 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:19,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741856_1032 (size=12207) 2024-11-10T19:16:19,782 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:19,782 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:19,782 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,782 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e23dc7ddc9ac44739d7f240de5dd35bd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/487c373511ac43c4978a74af134571c5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/16c1dec03cdf474c8fd40f1cf8f21b10] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=35.3 K 2024-11-10T19:16:19,784 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting e23dc7ddc9ac44739d7f240de5dd35bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731266176363 2024-11-10T19:16:19,784 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 487c373511ac43c4978a74af134571c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731266177096 2024-11-10T19:16:19,786 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16c1dec03cdf474c8fd40f1cf8f21b10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731266177519 2024-11-10T19:16:19,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266239778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:19,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741857_1033 (size=14341) 2024-11-10T19:16:19,822 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:19,823 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/3981ab2c416c4fafba049003e65409b8 is 50, key is test_row_0/C:col10/1731266178734/Put/seqid=0 2024-11-10T19:16:19,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741858_1034 (size=12207) 2024-11-10T19:16:19,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:19,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:19,884 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/3981ab2c416c4fafba049003e65409b8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3981ab2c416c4fafba049003e65409b8 2024-11-10T19:16:19,903 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 3981ab2c416c4fafba049003e65409b8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:19,904 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:19,904 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=13, startTime=1731266179605; duration=0sec 2024-11-10T19:16:19,904 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:19,905 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:19,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266239904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:20,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266240072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:20,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266240072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:20,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266240082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266240082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:20,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:20,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266240122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:20,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:20,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:20,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/8d4edd23a6eb4f5a95b9707b599db202 2024-11-10T19:16:20,206 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/6b209552aae9424bb1611415b7a24849 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/6b209552aae9424bb1611415b7a24849 2024-11-10T19:16:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T19:16:20,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/88d6e80b880e46daae9a5fc7a41e0822 is 50, key is test_row_0/B:col10/1731266179613/Put/seqid=0 2024-11-10T19:16:20,230 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 6b209552aae9424bb1611415b7a24849(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:20,230 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:20,231 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=13, startTime=1731266179605; duration=0sec 2024-11-10T19:16:20,232 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:20,232 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:20,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741859_1035 (size=12001) 2024-11-10T19:16:20,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/88d6e80b880e46daae9a5fc7a41e0822 2024-11-10T19:16:20,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/46f15fdf2e964530a3649533de605032 is 50, key is test_row_0/C:col10/1731266179613/Put/seqid=0 2024-11-10T19:16:20,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741860_1036 (size=12001) 2024-11-10T19:16:20,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266240441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:20,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:20,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:20,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:20,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/46f15fdf2e964530a3649533de605032 2024-11-10T19:16:20,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/8d4edd23a6eb4f5a95b9707b599db202 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8d4edd23a6eb4f5a95b9707b599db202 2024-11-10T19:16:20,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8d4edd23a6eb4f5a95b9707b599db202, entries=200, sequenceid=105, filesize=14.0 K 2024-11-10T19:16:20,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/88d6e80b880e46daae9a5fc7a41e0822 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/88d6e80b880e46daae9a5fc7a41e0822 2024-11-10T19:16:20,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/88d6e80b880e46daae9a5fc7a41e0822, entries=150, sequenceid=105, filesize=11.7 K 2024-11-10T19:16:20,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/46f15fdf2e964530a3649533de605032 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/46f15fdf2e964530a3649533de605032 2024-11-10T19:16:20,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/46f15fdf2e964530a3649533de605032, entries=150, sequenceid=105, filesize=11.7 K 2024-11-10T19:16:20,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 1196ms, sequenceid=105, compaction requested=false 2024-11-10T19:16:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:20,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:20,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:20,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:20,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/42447c89be0f437183302e2ac366dac7 is 50, key is test_row_0/A:col10/1731266179766/Put/seqid=0 2024-11-10T19:16:21,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741861_1037 (size=14341) 2024-11-10T19:16:21,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266241058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:21,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266241081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:21,089 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in 4f5e56bdcde85bbc88e3b9e5355a12b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:21,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266241091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:21,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266241092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:21,094 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 4f5e56bdcde85bbc88e3b9e5355a12b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:21,095 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in 4f5e56bdcde85bbc88e3b9e5355a12b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:21,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266241101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:21,103 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 4f5e56bdcde85bbc88e3b9e5355a12b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:21,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266241173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T19:16:21,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:21,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:21,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266241382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:21,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/42447c89be0f437183302e2ac366dac7 2024-11-10T19:16:21,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/6dbd68dcaf184e98b8101af3a54e1bcb is 50, key is test_row_0/B:col10/1731266179766/Put/seqid=0 2024-11-10T19:16:21,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:21,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:21,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741862_1038 (size=12001) 2024-11-10T19:16:21,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/6dbd68dcaf184e98b8101af3a54e1bcb 2024-11-10T19:16:21,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/d1d44b2a952d4c12877bee5145dab0f7 is 50, key is test_row_0/C:col10/1731266179766/Put/seqid=0 2024-11-10T19:16:21,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741863_1039 (size=12001) 2024-11-10T19:16:21,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/d1d44b2a952d4c12877bee5145dab0f7 2024-11-10T19:16:21,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/42447c89be0f437183302e2ac366dac7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/42447c89be0f437183302e2ac366dac7 2024-11-10T19:16:21,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/42447c89be0f437183302e2ac366dac7, entries=200, sequenceid=125, filesize=14.0 K 2024-11-10T19:16:21,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:21,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/6dbd68dcaf184e98b8101af3a54e1bcb as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6dbd68dcaf184e98b8101af3a54e1bcb 2024-11-10T19:16:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6dbd68dcaf184e98b8101af3a54e1bcb, entries=150, sequenceid=125, filesize=11.7 K 2024-11-10T19:16:21,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/d1d44b2a952d4c12877bee5145dab0f7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d1d44b2a952d4c12877bee5145dab0f7 2024-11-10T19:16:21,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d1d44b2a952d4c12877bee5145dab0f7, entries=150, sequenceid=125, filesize=11.7 K 2024-11-10T19:16:21,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 694ms, sequenceid=125, compaction requested=true 2024-11-10T19:16:21,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:21,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:21,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:21,651 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:21,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:21,651 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:21,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:21,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:21,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:21,653 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:21,654 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:21,654 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,654 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/6b209552aae9424bb1611415b7a24849, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8d4edd23a6eb4f5a95b9707b599db202, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/42447c89be0f437183302e2ac366dac7] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=39.9 K 2024-11-10T19:16:21,654 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:21,655 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:21,655 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,655 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/46c72a6b2c3f49f7bc7c448fb2013de6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/88d6e80b880e46daae9a5fc7a41e0822, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6dbd68dcaf184e98b8101af3a54e1bcb] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=35.4 K 2024-11-10T19:16:21,656 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 46c72a6b2c3f49f7bc7c448fb2013de6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731266177519 2024-11-10T19:16:21,656 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b209552aae9424bb1611415b7a24849, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731266177519 2024-11-10T19:16:21,656 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 88d6e80b880e46daae9a5fc7a41e0822, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731266178880 2024-11-10T19:16:21,657 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d4edd23a6eb4f5a95b9707b599db202, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731266178880 2024-11-10T19:16:21,658 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dbd68dcaf184e98b8101af3a54e1bcb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731266179754 2024-11-10T19:16:21,658 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42447c89be0f437183302e2ac366dac7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731266179742 2024-11-10T19:16:21,698 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:21,699 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/abfa7a3fcfda4e70aead0b967f9f01a3 is 50, key is test_row_0/B:col10/1731266179766/Put/seqid=0 2024-11-10T19:16:21,703 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:21,704 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/4ce7744d3611480f8fdbe440d1a78e01 is 50, key is test_row_0/A:col10/1731266179766/Put/seqid=0 2024-11-10T19:16:21,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:21,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:21,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/f310d8e122b240fbab9ffe173fa23a17 is 50, key is test_row_0/A:col10/1731266181707/Put/seqid=0 2024-11-10T19:16:21,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741865_1041 (size=12309) 2024-11-10T19:16:21,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741864_1040 (size=12309) 2024-11-10T19:16:21,795 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/abfa7a3fcfda4e70aead0b967f9f01a3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/abfa7a3fcfda4e70aead0b967f9f01a3 2024-11-10T19:16:21,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741866_1042 (size=14541) 2024-11-10T19:16:21,832 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into abfa7a3fcfda4e70aead0b967f9f01a3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:21,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:21,832 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=13, startTime=1731266181651; duration=0sec 2024-11-10T19:16:21,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:21,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:21,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:21,835 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:21,835 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:21,835 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,836 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3981ab2c416c4fafba049003e65409b8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/46f15fdf2e964530a3649533de605032, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d1d44b2a952d4c12877bee5145dab0f7] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=35.4 K 2024-11-10T19:16:21,836 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3981ab2c416c4fafba049003e65409b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731266177519 2024-11-10T19:16:21,837 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f15fdf2e964530a3649533de605032, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731266178880 2024-11-10T19:16:21,839 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting d1d44b2a952d4c12877bee5145dab0f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731266179754 2024-11-10T19:16:21,874 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:21,875 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/c639da8620204d51836565cab5129b58 is 50, key is test_row_0/C:col10/1731266179766/Put/seqid=0 2024-11-10T19:16:21,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741867_1043 (size=12309) 2024-11-10T19:16:21,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:21,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:21,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:21,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:21,935 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/c639da8620204d51836565cab5129b58 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/c639da8620204d51836565cab5129b58 2024-11-10T19:16:21,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266241932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:21,949 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into c639da8620204d51836565cab5129b58(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:21,949 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:21,949 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=13, startTime=1731266181651; duration=0sec 2024-11-10T19:16:21,949 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:21,949 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:22,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266242052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:22,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:22,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:22,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:22,218 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/4ce7744d3611480f8fdbe440d1a78e01 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4ce7744d3611480f8fdbe440d1a78e01 2024-11-10T19:16:22,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/f310d8e122b240fbab9ffe173fa23a17 2024-11-10T19:16:22,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:22,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:22,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:22,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:22,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:22,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:22,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/fd68de2b5a044285a1e2772a7a4dead2 is 50, key is test_row_0/B:col10/1731266181707/Put/seqid=0 2024-11-10T19:16:22,258 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 4ce7744d3611480f8fdbe440d1a78e01(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:22,258 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:22,258 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=13, startTime=1731266181651; duration=0sec 2024-11-10T19:16:22,259 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:22,259 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:22,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266242265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:22,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741868_1044 (size=12151) 2024-11-10T19:16:22,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/fd68de2b5a044285a1e2772a7a4dead2 2024-11-10T19:16:22,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/4dec4dd39e884375a48bd58acc0f0c16 is 50, key is test_row_0/C:col10/1731266181707/Put/seqid=0 2024-11-10T19:16:22,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741869_1045 (size=12151) 2024-11-10T19:16:22,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/4dec4dd39e884375a48bd58acc0f0c16 2024-11-10T19:16:22,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/f310d8e122b240fbab9ffe173fa23a17 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/f310d8e122b240fbab9ffe173fa23a17 2024-11-10T19:16:22,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:22,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:22,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:22,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:22,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:22,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:22,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/f310d8e122b240fbab9ffe173fa23a17, entries=200, sequenceid=141, filesize=14.2 K 2024-11-10T19:16:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:22,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/fd68de2b5a044285a1e2772a7a4dead2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/fd68de2b5a044285a1e2772a7a4dead2 2024-11-10T19:16:22,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/fd68de2b5a044285a1e2772a7a4dead2, entries=150, sequenceid=141, filesize=11.9 K 2024-11-10T19:16:22,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/4dec4dd39e884375a48bd58acc0f0c16 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4dec4dd39e884375a48bd58acc0f0c16 2024-11-10T19:16:22,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4dec4dd39e884375a48bd58acc0f0c16, entries=150, sequenceid=141, filesize=11.9 K 2024-11-10T19:16:22,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 728ms, sequenceid=141, compaction requested=false 2024-11-10T19:16:22,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:22,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T19:16:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:22,553 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:22,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:22,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a5414d69e6994321834748a709eb549d is 50, key is test_row_0/A:col10/1731266181825/Put/seqid=0 2024-11-10T19:16:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:22,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:22,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741870_1046 (size=12151) 2024-11-10T19:16:22,652 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a5414d69e6994321834748a709eb549d 2024-11-10T19:16:22,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266242685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/66b6646fd8764dde8fe009909589c3c8 is 50, key is test_row_0/B:col10/1731266181825/Put/seqid=0 2024-11-10T19:16:22,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741871_1047 (size=12151) 2024-11-10T19:16:22,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266242801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:23,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266243011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:23,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266243102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:23,112 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in 4f5e56bdcde85bbc88e3b9e5355a12b5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:23,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266243111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:23,118 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in 4f5e56bdcde85bbc88e3b9e5355a12b5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:23,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266243111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:23,124 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 4f5e56bdcde85bbc88e3b9e5355a12b5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:23,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266243132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:23,138 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 4f5e56bdcde85bbc88e3b9e5355a12b5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:23,145 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/66b6646fd8764dde8fe009909589c3c8 2024-11-10T19:16:23,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/85984b7e0c904a7e982f7e2c0898b7ba is 50, key is test_row_0/C:col10/1731266181825/Put/seqid=0 2024-11-10T19:16:23,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741872_1048 (size=12151) 2024-11-10T19:16:23,175 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/85984b7e0c904a7e982f7e2c0898b7ba 2024-11-10T19:16:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a5414d69e6994321834748a709eb549d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a5414d69e6994321834748a709eb549d 2024-11-10T19:16:23,212 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a5414d69e6994321834748a709eb549d, entries=150, sequenceid=160, filesize=11.9 K 2024-11-10T19:16:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/66b6646fd8764dde8fe009909589c3c8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/66b6646fd8764dde8fe009909589c3c8 2024-11-10T19:16:23,226 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/66b6646fd8764dde8fe009909589c3c8, entries=150, sequenceid=160, filesize=11.9 K 2024-11-10T19:16:23,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/85984b7e0c904a7e982f7e2c0898b7ba as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/85984b7e0c904a7e982f7e2c0898b7ba 2024-11-10T19:16:23,236 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/85984b7e0c904a7e982f7e2c0898b7ba, entries=150, sequenceid=160, filesize=11.9 K 2024-11-10T19:16:23,238 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 685ms, sequenceid=160, compaction requested=true 2024-11-10T19:16:23,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:23,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:23,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T19:16:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T19:16:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T19:16:23,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T19:16:23,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1720 sec 2024-11-10T19:16:23,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 4.1880 sec 2024-11-10T19:16:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:23,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:23,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:23,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:23,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:23,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:23,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:23,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:23,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/5c637dce15d349b29daf37fa2fc7005e is 50, key is test_row_0/A:col10/1731266182664/Put/seqid=0 2024-11-10T19:16:23,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741873_1049 (size=14541) 2024-11-10T19:16:23,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/5c637dce15d349b29daf37fa2fc7005e 2024-11-10T19:16:23,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/992a3e901d5644b0a5d26774d040845d is 50, key is test_row_0/B:col10/1731266182664/Put/seqid=0 2024-11-10T19:16:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266243437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:23,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741874_1050 (size=12151) 2024-11-10T19:16:23,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266243552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:23,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266243762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:23,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/992a3e901d5644b0a5d26774d040845d 2024-11-10T19:16:23,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/2c7b963df45c4026848a6b8af169dcc1 is 50, key is test_row_0/C:col10/1731266182664/Put/seqid=0 2024-11-10T19:16:23,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741875_1051 (size=12151) 2024-11-10T19:16:24,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266244082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:24,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/2c7b963df45c4026848a6b8af169dcc1 2024-11-10T19:16:24,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/5c637dce15d349b29daf37fa2fc7005e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5c637dce15d349b29daf37fa2fc7005e 2024-11-10T19:16:24,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5c637dce15d349b29daf37fa2fc7005e, entries=200, sequenceid=177, filesize=14.2 K 2024-11-10T19:16:24,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/992a3e901d5644b0a5d26774d040845d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/992a3e901d5644b0a5d26774d040845d 2024-11-10T19:16:24,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/992a3e901d5644b0a5d26774d040845d, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T19:16:24,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/2c7b963df45c4026848a6b8af169dcc1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/2c7b963df45c4026848a6b8af169dcc1 2024-11-10T19:16:24,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/2c7b963df45c4026848a6b8af169dcc1, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T19:16:24,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 1036ms, sequenceid=177, compaction requested=true 2024-11-10T19:16:24,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:24,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:24,374 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:16:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:24,374 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:16:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:24,375 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:16:24,376 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:24,376 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:24,376 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4ce7744d3611480f8fdbe440d1a78e01, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/f310d8e122b240fbab9ffe173fa23a17, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a5414d69e6994321834748a709eb549d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5c637dce15d349b29daf37fa2fc7005e] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=52.3 K 2024-11-10T19:16:24,376 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:16:24,376 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ce7744d3611480f8fdbe440d1a78e01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731266179754 2024-11-10T19:16:24,377 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:24,377 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:24,377 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/abfa7a3fcfda4e70aead0b967f9f01a3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/fd68de2b5a044285a1e2772a7a4dead2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/66b6646fd8764dde8fe009909589c3c8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/992a3e901d5644b0a5d26774d040845d] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=47.6 K 2024-11-10T19:16:24,378 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f310d8e122b240fbab9ffe173fa23a17, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731266181007 2024-11-10T19:16:24,378 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting abfa7a3fcfda4e70aead0b967f9f01a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731266179754 2024-11-10T19:16:24,378 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting fd68de2b5a044285a1e2772a7a4dead2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731266181007 2024-11-10T19:16:24,378 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5414d69e6994321834748a709eb549d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731266181825 2024-11-10T19:16:24,379 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 66b6646fd8764dde8fe009909589c3c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731266181825 2024-11-10T19:16:24,380 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c637dce15d349b29daf37fa2fc7005e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731266182655 2024-11-10T19:16:24,380 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 992a3e901d5644b0a5d26774d040845d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731266182664 2024-11-10T19:16:24,412 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:24,413 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/b602d7b3a62e4a099c456e1ceacebdcb is 50, key is test_row_0/B:col10/1731266182664/Put/seqid=0 2024-11-10T19:16:24,420 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:24,422 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/0e85da0a02364c419c6421fe6ea10211 is 50, key is test_row_0/A:col10/1731266182664/Put/seqid=0 2024-11-10T19:16:24,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741876_1052 (size=12595) 2024-11-10T19:16:24,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741877_1053 (size=12595) 2024-11-10T19:16:24,462 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/b602d7b3a62e4a099c456e1ceacebdcb as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/b602d7b3a62e4a099c456e1ceacebdcb 2024-11-10T19:16:24,480 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/0e85da0a02364c419c6421fe6ea10211 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0e85da0a02364c419c6421fe6ea10211 2024-11-10T19:16:24,482 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into b602d7b3a62e4a099c456e1ceacebdcb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:24,482 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:24,482 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=12, startTime=1731266184374; duration=0sec 2024-11-10T19:16:24,483 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:24,483 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:24,483 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:16:24,485 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:16:24,485 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:24,486 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:24,486 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/c639da8620204d51836565cab5129b58, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4dec4dd39e884375a48bd58acc0f0c16, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/85984b7e0c904a7e982f7e2c0898b7ba, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/2c7b963df45c4026848a6b8af169dcc1] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=47.6 K 2024-11-10T19:16:24,487 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c639da8620204d51836565cab5129b58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731266179754 2024-11-10T19:16:24,488 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dec4dd39e884375a48bd58acc0f0c16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731266181007 2024-11-10T19:16:24,491 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 85984b7e0c904a7e982f7e2c0898b7ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731266181825 2024-11-10T19:16:24,493 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c7b963df45c4026848a6b8af169dcc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731266182664 2024-11-10T19:16:24,493 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 0e85da0a02364c419c6421fe6ea10211(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:24,493 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:24,493 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=12, startTime=1731266184373; duration=0sec 2024-11-10T19:16:24,493 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:24,493 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:24,513 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#41 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:24,514 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/82a00f727db749c580170f84db3e4a4d is 50, key is test_row_0/C:col10/1731266182664/Put/seqid=0 2024-11-10T19:16:24,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741878_1054 (size=12595) 2024-11-10T19:16:24,557 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/82a00f727db749c580170f84db3e4a4d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/82a00f727db749c580170f84db3e4a4d 2024-11-10T19:16:24,575 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 82a00f727db749c580170f84db3e4a4d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:24,575 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:24,575 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=12, startTime=1731266184374; duration=0sec 2024-11-10T19:16:24,576 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:24,576 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:24,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:24,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:24,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:24,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:24,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:24,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:24,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:24,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/14192518050b4ba4a7cd3ec93f112042 is 50, key is test_row_0/A:col10/1731266183344/Put/seqid=0 2024-11-10T19:16:24,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741879_1055 (size=12151) 2024-11-10T19:16:24,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/14192518050b4ba4a7cd3ec93f112042 2024-11-10T19:16:24,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/2eb47d989051461fb633e7c62f9d8e6e is 50, key is test_row_0/B:col10/1731266183344/Put/seqid=0 2024-11-10T19:16:24,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741880_1056 (size=12151) 2024-11-10T19:16:24,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/2eb47d989051461fb633e7c62f9d8e6e 2024-11-10T19:16:24,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/83c318f239b14838a17cf8f38d97a3fc is 50, key is test_row_0/C:col10/1731266183344/Put/seqid=0 2024-11-10T19:16:24,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741881_1057 (size=12151) 2024-11-10T19:16:24,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266244725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:24,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266244842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:25,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266245051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:25,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/83c318f239b14838a17cf8f38d97a3fc 2024-11-10T19:16:25,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/14192518050b4ba4a7cd3ec93f112042 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/14192518050b4ba4a7cd3ec93f112042 2024-11-10T19:16:25,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/14192518050b4ba4a7cd3ec93f112042, entries=150, sequenceid=196, filesize=11.9 K 2024-11-10T19:16:25,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/2eb47d989051461fb633e7c62f9d8e6e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb47d989051461fb633e7c62f9d8e6e 2024-11-10T19:16:25,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb47d989051461fb633e7c62f9d8e6e, entries=150, sequenceid=196, filesize=11.9 K 2024-11-10T19:16:25,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/83c318f239b14838a17cf8f38d97a3fc as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/83c318f239b14838a17cf8f38d97a3fc 2024-11-10T19:16:25,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/83c318f239b14838a17cf8f38d97a3fc, entries=150, sequenceid=196, filesize=11.9 K 2024-11-10T19:16:25,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 622ms, sequenceid=196, compaction requested=false 2024-11-10T19:16:25,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:25,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:25,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:25,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/c2e1f77ed4734126bd3079bd34cb9d38 is 50, key is test_row_0/A:col10/1731266185372/Put/seqid=0 2024-11-10T19:16:25,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741882_1058 (size=14541) 2024-11-10T19:16:25,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/c2e1f77ed4734126bd3079bd34cb9d38 2024-11-10T19:16:25,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/6ef81d2747284a96bcd82a240a9657a4 is 50, key is test_row_0/B:col10/1731266185372/Put/seqid=0 2024-11-10T19:16:25,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266245451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:25,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741883_1059 (size=12151) 2024-11-10T19:16:25,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/6ef81d2747284a96bcd82a240a9657a4 2024-11-10T19:16:25,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/4c330065c1934320825bbd5033a88c89 is 50, key is test_row_0/C:col10/1731266185372/Put/seqid=0 2024-11-10T19:16:25,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741884_1060 (size=12151) 2024-11-10T19:16:25,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/4c330065c1934320825bbd5033a88c89 2024-11-10T19:16:25,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/c2e1f77ed4734126bd3079bd34cb9d38 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c2e1f77ed4734126bd3079bd34cb9d38 2024-11-10T19:16:25,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c2e1f77ed4734126bd3079bd34cb9d38, entries=200, sequenceid=213, filesize=14.2 K 2024-11-10T19:16:25,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/6ef81d2747284a96bcd82a240a9657a4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6ef81d2747284a96bcd82a240a9657a4 2024-11-10T19:16:25,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6ef81d2747284a96bcd82a240a9657a4, entries=150, sequenceid=213, filesize=11.9 K 2024-11-10T19:16:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/4c330065c1934320825bbd5033a88c89 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4c330065c1934320825bbd5033a88c89 2024-11-10T19:16:25,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4c330065c1934320825bbd5033a88c89, entries=150, sequenceid=213, filesize=11.9 K 2024-11-10T19:16:25,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 164ms, sequenceid=213, compaction requested=true 2024-11-10T19:16:25,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:25,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:25,538 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:25,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:25,538 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:25,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:25,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:25,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:25,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:25,540 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:25,540 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:25,541 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:25,541 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/b602d7b3a62e4a099c456e1ceacebdcb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb47d989051461fb633e7c62f9d8e6e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6ef81d2747284a96bcd82a240a9657a4] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=36.0 K 2024-11-10T19:16:25,541 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:25,541 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:25,541 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:25,541 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0e85da0a02364c419c6421fe6ea10211, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/14192518050b4ba4a7cd3ec93f112042, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c2e1f77ed4734126bd3079bd34cb9d38] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=38.4 K 2024-11-10T19:16:25,542 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e85da0a02364c419c6421fe6ea10211, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731266182664 2024-11-10T19:16:25,542 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting b602d7b3a62e4a099c456e1ceacebdcb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731266182664 2024-11-10T19:16:25,543 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14192518050b4ba4a7cd3ec93f112042, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731266183344 2024-11-10T19:16:25,543 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eb47d989051461fb633e7c62f9d8e6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731266183344 2024-11-10T19:16:25,544 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e1f77ed4734126bd3079bd34cb9d38, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731266184680 2024-11-10T19:16:25,548 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef81d2747284a96bcd82a240a9657a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731266184680 2024-11-10T19:16:25,574 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:25,575 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/1236070c1b5b4048990c849904812eb5 is 50, key is test_row_0/B:col10/1731266185372/Put/seqid=0 2024-11-10T19:16:25,576 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#48 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:25,577 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/d0a1c385c00941f2acdc108969742bd1 is 50, key is test_row_0/A:col10/1731266185372/Put/seqid=0 2024-11-10T19:16:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:25,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:25,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/861a4e40ffe04a6e910c51c1cab9c722 is 50, key is test_row_0/A:col10/1731266185442/Put/seqid=0 2024-11-10T19:16:25,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741885_1061 (size=12697) 2024-11-10T19:16:25,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741886_1062 (size=12697) 2024-11-10T19:16:25,642 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/1236070c1b5b4048990c849904812eb5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1236070c1b5b4048990c849904812eb5 2024-11-10T19:16:25,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741887_1063 (size=12151) 2024-11-10T19:16:25,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/861a4e40ffe04a6e910c51c1cab9c722 2024-11-10T19:16:25,657 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 1236070c1b5b4048990c849904812eb5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:25,658 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:25,658 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=13, startTime=1731266185538; duration=0sec 2024-11-10T19:16:25,658 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:25,658 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:25,658 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:25,664 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:25,664 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:25,664 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:25,664 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/82a00f727db749c580170f84db3e4a4d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/83c318f239b14838a17cf8f38d97a3fc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4c330065c1934320825bbd5033a88c89] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=36.0 K 2024-11-10T19:16:25,667 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 82a00f727db749c580170f84db3e4a4d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731266182664 2024-11-10T19:16:25,667 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 83c318f239b14838a17cf8f38d97a3fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731266183344 2024-11-10T19:16:25,668 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c330065c1934320825bbd5033a88c89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731266184680 2024-11-10T19:16:25,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/06c75f666e97446e818b92e4bf3070ce is 50, key is test_row_0/B:col10/1731266185442/Put/seqid=0 2024-11-10T19:16:25,683 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:25,684 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/5201823e57d643fa9983fde84241a425 is 50, key is test_row_0/C:col10/1731266185372/Put/seqid=0 2024-11-10T19:16:25,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266245720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:25,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:25,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:25,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:25,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741889_1065 (size=12697) 2024-11-10T19:16:25,747 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/5201823e57d643fa9983fde84241a425 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5201823e57d643fa9983fde84241a425 2024-11-10T19:16:25,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741888_1064 (size=12151) 2024-11-10T19:16:25,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/06c75f666e97446e818b92e4bf3070ce 2024-11-10T19:16:25,771 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 5201823e57d643fa9983fde84241a425(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:25,771 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:25,771 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=13, startTime=1731266185539; duration=0sec 2024-11-10T19:16:25,771 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:25,771 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:25,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/69196693a2144258898ec07e58038b55 is 50, key is test_row_0/C:col10/1731266185442/Put/seqid=0 2024-11-10T19:16:25,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741890_1066 (size=12151) 2024-11-10T19:16:25,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266245832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:26,035 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/d0a1c385c00941f2acdc108969742bd1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d0a1c385c00941f2acdc108969742bd1 2024-11-10T19:16:26,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266246044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:26,047 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into d0a1c385c00941f2acdc108969742bd1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:26,047 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:26,047 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=13, startTime=1731266185538; duration=0sec 2024-11-10T19:16:26,047 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:26,047 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:26,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/69196693a2144258898ec07e58038b55 2024-11-10T19:16:26,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/861a4e40ffe04a6e910c51c1cab9c722 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/861a4e40ffe04a6e910c51c1cab9c722 2024-11-10T19:16:26,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/861a4e40ffe04a6e910c51c1cab9c722, entries=150, sequenceid=229, filesize=11.9 K 2024-11-10T19:16:26,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/06c75f666e97446e818b92e4bf3070ce as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06c75f666e97446e818b92e4bf3070ce 2024-11-10T19:16:26,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06c75f666e97446e818b92e4bf3070ce, entries=150, sequenceid=229, filesize=11.9 K 2024-11-10T19:16:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/69196693a2144258898ec07e58038b55 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/69196693a2144258898ec07e58038b55 2024-11-10T19:16:26,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/69196693a2144258898ec07e58038b55, entries=150, sequenceid=229, filesize=11.9 K 2024-11-10T19:16:26,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 688ms, sequenceid=229, compaction requested=false 2024-11-10T19:16:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:26,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:26,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/df10501395b54a11871ac52cffaec306 is 50, key is test_row_0/A:col10/1731266185655/Put/seqid=0 2024-11-10T19:16:26,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741891_1067 (size=12151) 2024-11-10T19:16:26,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/df10501395b54a11871ac52cffaec306 2024-11-10T19:16:26,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/091618cfb0dc43f087dee50253839ad4 is 50, key is test_row_0/B:col10/1731266185655/Put/seqid=0 2024-11-10T19:16:26,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741892_1068 (size=12151) 2024-11-10T19:16:26,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/091618cfb0dc43f087dee50253839ad4 2024-11-10T19:16:26,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/9ba5b43d22ef4c78b4f40aa3e6901f50 is 50, key is test_row_0/C:col10/1731266185655/Put/seqid=0 2024-11-10T19:16:26,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266246473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:26,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741893_1069 (size=12151) 2024-11-10T19:16:26,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266246593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:26,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266246802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:26,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/9ba5b43d22ef4c78b4f40aa3e6901f50 2024-11-10T19:16:26,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/df10501395b54a11871ac52cffaec306 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/df10501395b54a11871ac52cffaec306 2024-11-10T19:16:26,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/df10501395b54a11871ac52cffaec306, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T19:16:26,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/091618cfb0dc43f087dee50253839ad4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/091618cfb0dc43f087dee50253839ad4 2024-11-10T19:16:26,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/091618cfb0dc43f087dee50253839ad4, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T19:16:26,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/9ba5b43d22ef4c78b4f40aa3e6901f50 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9ba5b43d22ef4c78b4f40aa3e6901f50 2024-11-10T19:16:26,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9ba5b43d22ef4c78b4f40aa3e6901f50, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T19:16:26,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 599ms, sequenceid=249, compaction requested=true 2024-11-10T19:16:26,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:26,960 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:26,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:26,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:26,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:26,960 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:26,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:26,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:26,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:26,962 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:26,962 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:26,962 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:26,962 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d0a1c385c00941f2acdc108969742bd1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/861a4e40ffe04a6e910c51c1cab9c722, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/df10501395b54a11871ac52cffaec306] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=36.1 K 2024-11-10T19:16:26,963 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:26,963 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a1c385c00941f2acdc108969742bd1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731266184680 2024-11-10T19:16:26,963 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:26,963 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:26,963 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1236070c1b5b4048990c849904812eb5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06c75f666e97446e818b92e4bf3070ce, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/091618cfb0dc43f087dee50253839ad4] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=36.1 K 2024-11-10T19:16:26,963 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 861a4e40ffe04a6e910c51c1cab9c722, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731266185442 2024-11-10T19:16:26,964 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 1236070c1b5b4048990c849904812eb5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731266184680 2024-11-10T19:16:26,964 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 06c75f666e97446e818b92e4bf3070ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731266185442 2024-11-10T19:16:26,965 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting df10501395b54a11871ac52cffaec306, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731266185655 2024-11-10T19:16:26,965 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 091618cfb0dc43f087dee50253839ad4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731266185655 2024-11-10T19:16:27,007 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:27,008 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/5732939e2acf4e6597b6ff889ffd0a47 is 50, key is test_row_0/B:col10/1731266185655/Put/seqid=0 2024-11-10T19:16:27,023 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:27,024 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/8098ba351ac24036a66573ed1eaf1ee4 is 50, key is test_row_0/A:col10/1731266185655/Put/seqid=0 2024-11-10T19:16:27,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741894_1070 (size=12799) 2024-11-10T19:16:27,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741895_1071 (size=12799) 2024-11-10T19:16:27,099 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/5732939e2acf4e6597b6ff889ffd0a47 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/5732939e2acf4e6597b6ff889ffd0a47 2024-11-10T19:16:27,107 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/8098ba351ac24036a66573ed1eaf1ee4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8098ba351ac24036a66573ed1eaf1ee4 2024-11-10T19:16:27,118 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 5732939e2acf4e6597b6ff889ffd0a47(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:27,118 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:27,118 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=13, startTime=1731266186960; duration=0sec 2024-11-10T19:16:27,118 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:27,118 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:27,120 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:27,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:27,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:27,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:27,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:27,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:27,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:27,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:27,124 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 8098ba351ac24036a66573ed1eaf1ee4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:27,124 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:27,124 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=13, startTime=1731266186960; duration=0sec 2024-11-10T19:16:27,125 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:27,125 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:27,130 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:27,131 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:27,131 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:27,132 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5201823e57d643fa9983fde84241a425, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/69196693a2144258898ec07e58038b55, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9ba5b43d22ef4c78b4f40aa3e6901f50] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=36.1 K 2024-11-10T19:16:27,133 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5201823e57d643fa9983fde84241a425, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731266184680 2024-11-10T19:16:27,134 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 69196693a2144258898ec07e58038b55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731266185442 2024-11-10T19:16:27,135 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba5b43d22ef4c78b4f40aa3e6901f50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731266185655 2024-11-10T19:16:27,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/0bd5b55125674f339416a61adb3546cf is 50, key is test_row_0/A:col10/1731266186449/Put/seqid=0 2024-11-10T19:16:27,177 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:27,178 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/90d4cf0876084b6d96bb1185a505798c is 50, key is test_row_0/C:col10/1731266185655/Put/seqid=0 2024-11-10T19:16:27,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266247164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266247179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266247179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266247180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266247187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741896_1072 (size=12297) 2024-11-10T19:16:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741897_1073 (size=12799) 2024-11-10T19:16:27,223 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/90d4cf0876084b6d96bb1185a505798c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/90d4cf0876084b6d96bb1185a505798c 2024-11-10T19:16:27,231 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 90d4cf0876084b6d96bb1185a505798c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:27,232 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:27,232 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=13, startTime=1731266186960; duration=0sec 2024-11-10T19:16:27,232 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:27,232 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T19:16:27,265 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:27,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T19:16:27,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T19:16:27,269 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:27,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:27,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266247302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266247302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266247302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266247302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266247302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T19:16:27,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T19:16:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:27,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266247511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266247511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266247512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266247512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266247512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T19:16:27,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:27,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:27,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:27,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T19:16:27,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/0bd5b55125674f339416a61adb3546cf 2024-11-10T19:16:27,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/3f0ed730506f4658a72b9d6a1d5ead24 is 50, key is test_row_0/B:col10/1731266186449/Put/seqid=0 2024-11-10T19:16:27,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741898_1074 (size=9857) 2024-11-10T19:16:27,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T19:16:27,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:27,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:27,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:27,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266247821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266247821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266247821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266247822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266247822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:27,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T19:16:27,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:27,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:27,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:27,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T19:16:28,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/3f0ed730506f4658a72b9d6a1d5ead24 2024-11-10T19:16:28,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/92e4defbd18a4d5a971366d3e6a9b30f is 50, key is test_row_0/C:col10/1731266186449/Put/seqid=0 2024-11-10T19:16:28,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741899_1075 (size=9857) 2024-11-10T19:16:28,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T19:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:28,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:28,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T19:16:28,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:28,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:28,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:28,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:28,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:28,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266248332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:28,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:28,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266248332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:28,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:28,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266248332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:28,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:28,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266248332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266248332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:28,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T19:16:28,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:28,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:28,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:28,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:28,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T19:16:28,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/92e4defbd18a4d5a971366d3e6a9b30f 2024-11-10T19:16:28,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/0bd5b55125674f339416a61adb3546cf as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0bd5b55125674f339416a61adb3546cf 2024-11-10T19:16:28,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0bd5b55125674f339416a61adb3546cf, entries=150, sequenceid=266, filesize=12.0 K 2024-11-10T19:16:28,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/3f0ed730506f4658a72b9d6a1d5ead24 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3f0ed730506f4658a72b9d6a1d5ead24 2024-11-10T19:16:28,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3f0ed730506f4658a72b9d6a1d5ead24, entries=100, sequenceid=266, filesize=9.6 K 2024-11-10T19:16:28,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/92e4defbd18a4d5a971366d3e6a9b30f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/92e4defbd18a4d5a971366d3e6a9b30f 2024-11-10T19:16:28,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/92e4defbd18a4d5a971366d3e6a9b30f, entries=100, sequenceid=266, filesize=9.6 K 2024-11-10T19:16:28,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 1345ms, sequenceid=266, compaction requested=false 2024-11-10T19:16:28,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:28,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T19:16:28,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:28,513 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:28,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/9ce110cb88b0422c9af762c350d0d1c5 is 50, key is test_row_0/A:col10/1731266187174/Put/seqid=0 2024-11-10T19:16:28,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741900_1076 (size=12301) 2024-11-10T19:16:28,927 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/9ce110cb88b0422c9af762c350d0d1c5 2024-11-10T19:16:28,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/06dc92b5144d4099a422ae510819dc0c is 50, key is test_row_0/B:col10/1731266187174/Put/seqid=0 2024-11-10T19:16:28,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741901_1077 (size=12301) 2024-11-10T19:16:28,965 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/06dc92b5144d4099a422ae510819dc0c 2024-11-10T19:16:28,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/674d18e4edb14b80af3ae7b7797f1785 is 50, key is test_row_0/C:col10/1731266187174/Put/seqid=0 2024-11-10T19:16:29,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741902_1078 (size=12301) 2024-11-10T19:16:29,021 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/674d18e4edb14b80af3ae7b7797f1785 2024-11-10T19:16:29,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/9ce110cb88b0422c9af762c350d0d1c5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9ce110cb88b0422c9af762c350d0d1c5 2024-11-10T19:16:29,042 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9ce110cb88b0422c9af762c350d0d1c5, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T19:16:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/06dc92b5144d4099a422ae510819dc0c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06dc92b5144d4099a422ae510819dc0c 2024-11-10T19:16:29,053 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06dc92b5144d4099a422ae510819dc0c, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T19:16:29,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/674d18e4edb14b80af3ae7b7797f1785 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/674d18e4edb14b80af3ae7b7797f1785 2024-11-10T19:16:29,069 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/674d18e4edb14b80af3ae7b7797f1785, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T19:16:29,071 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 558ms, sequenceid=284, compaction requested=true 2024-11-10T19:16:29,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:29,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T19:16:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T19:16:29,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T19:16:29,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-11-10T19:16:29,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.8120 sec 2024-11-10T19:16:29,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:16:29,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:29,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:29,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:29,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:29,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:29,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:29,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/bcb6289f1bf145c78a452547df114e5e is 50, key is test_row_0/A:col10/1731266189356/Put/seqid=0 2024-11-10T19:16:29,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741903_1079 (size=14741) 2024-11-10T19:16:29,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/bcb6289f1bf145c78a452547df114e5e 2024-11-10T19:16:29,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/43ac8c96ed4a412681dea73d0fc2d6df is 50, key is test_row_0/B:col10/1731266189356/Put/seqid=0 2024-11-10T19:16:29,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266249387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266249389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266249390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266249393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266249399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731266189376, max timestamp=1731266189392Num uniques 859; 2024-11-10T19:16:29,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T19:16:29,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731266189376, max timestamp=1731266189392Num uniques 859; 2024-11-10T19:16:29,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741904_1080 (size=12301) 2024-11-10T19:16:29,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731266189376, max timestamp=1731266189392Num uniques 859; 2024-11-10T19:16:29,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/43ac8c96ed4a412681dea73d0fc2d6df 2024-11-10T19:16:29,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T19:16:29,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T19:16:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T19:16:29,412 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:29,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T19:16:29,418 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T19:16:29,419 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:29,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:29,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/dd69442c4b694be58c27263e177e2b2a is 50, key is test_row_0/C:col10/1731266189356/Put/seqid=0 2024-11-10T19:16:29,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741905_1081 (size=12301) 2024-11-10T19:16:29,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/dd69442c4b694be58c27263e177e2b2a 2024-11-10T19:16:29,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/bcb6289f1bf145c78a452547df114e5e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/bcb6289f1bf145c78a452547df114e5e 2024-11-10T19:16:29,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/bcb6289f1bf145c78a452547df114e5e, entries=200, sequenceid=295, filesize=14.4 K 2024-11-10T19:16:29,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/43ac8c96ed4a412681dea73d0fc2d6df as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43ac8c96ed4a412681dea73d0fc2d6df 2024-11-10T19:16:29,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43ac8c96ed4a412681dea73d0fc2d6df, entries=150, sequenceid=295, filesize=12.0 K 2024-11-10T19:16:29,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/dd69442c4b694be58c27263e177e2b2a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dd69442c4b694be58c27263e177e2b2a 2024-11-10T19:16:29,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dd69442c4b694be58c27263e177e2b2a, entries=150, sequenceid=295, filesize=12.0 K 2024-11-10T19:16:29,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=60.38 KB/61830 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 130ms, sequenceid=295, compaction requested=true 2024-11-10T19:16:29,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:29,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:29,487 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:16:29,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:29,488 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:16:29,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:29,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:29,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:29,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:29,490 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:16:29,490 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:29,490 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,490 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52138 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:16:29,490 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:29,490 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/5732939e2acf4e6597b6ff889ffd0a47, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3f0ed730506f4658a72b9d6a1d5ead24, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06dc92b5144d4099a422ae510819dc0c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43ac8c96ed4a412681dea73d0fc2d6df] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=46.2 K 2024-11-10T19:16:29,490 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,491 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8098ba351ac24036a66573ed1eaf1ee4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0bd5b55125674f339416a61adb3546cf, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9ce110cb88b0422c9af762c350d0d1c5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/bcb6289f1bf145c78a452547df114e5e] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=50.9 K 2024-11-10T19:16:29,491 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5732939e2acf4e6597b6ff889ffd0a47, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731266185655 2024-11-10T19:16:29,491 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8098ba351ac24036a66573ed1eaf1ee4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731266185655 2024-11-10T19:16:29,492 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd5b55125674f339416a61adb3546cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731266186449 2024-11-10T19:16:29,492 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0ed730506f4658a72b9d6a1d5ead24, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731266186449 2024-11-10T19:16:29,492 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ce110cb88b0422c9af762c350d0d1c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731266187174 2024-11-10T19:16:29,493 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 06dc92b5144d4099a422ae510819dc0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731266187174 2024-11-10T19:16:29,493 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcb6289f1bf145c78a452547df114e5e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731266189348 2024-11-10T19:16:29,495 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ac8c96ed4a412681dea73d0fc2d6df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731266189355 2024-11-10T19:16:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:29,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=67.09 KB heapSize=162.33 KB 2024-11-10T19:16:29,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:29,519 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:29,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:16:29,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:29,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:16:29,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:29,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:16:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T19:16:29,521 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/5d3c6f23d49e408181db585b74b0c3f4 is 50, key is test_row_0/A:col10/1731266189356/Put/seqid=0 2024-11-10T19:16:29,524 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:29,525 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/35b760ddc09e45e182135a11c4daf166 is 50, key is test_row_0/B:col10/1731266189356/Put/seqid=0 2024-11-10T19:16:29,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/4bf770ca5c724cecb95ca32f47713434 is 50, key is test_row_0/A:col10/1731266189392/Put/seqid=0 2024-11-10T19:16:29,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741906_1082 (size=13085) 2024-11-10T19:16:29,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741907_1083 (size=13085) 2024-11-10T19:16:29,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T19:16:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:29,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:29,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266249562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:29,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266249565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266249569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266249571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266249577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,585 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/35b760ddc09e45e182135a11c4daf166 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/35b760ddc09e45e182135a11c4daf166 2024-11-10T19:16:29,586 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/5d3c6f23d49e408181db585b74b0c3f4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5d3c6f23d49e408181db585b74b0c3f4 2024-11-10T19:16:29,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741908_1084 (size=12301) 2024-11-10T19:16:29,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/4bf770ca5c724cecb95ca32f47713434 2024-11-10T19:16:29,598 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 35b760ddc09e45e182135a11c4daf166(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:29,598 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:29,598 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=12, startTime=1731266189488; duration=0sec 2024-11-10T19:16:29,598 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:29,598 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:29,599 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:16:29,603 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 5d3c6f23d49e408181db585b74b0c3f4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:29,603 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:29,603 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=12, startTime=1731266189487; duration=0sec 2024-11-10T19:16:29,604 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:29,604 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:29,605 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:16:29,605 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:29,606 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,606 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/90d4cf0876084b6d96bb1185a505798c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/92e4defbd18a4d5a971366d3e6a9b30f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/674d18e4edb14b80af3ae7b7797f1785, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dd69442c4b694be58c27263e177e2b2a] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=46.2 K 2024-11-10T19:16:29,609 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 90d4cf0876084b6d96bb1185a505798c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731266185655 2024-11-10T19:16:29,609 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e4defbd18a4d5a971366d3e6a9b30f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731266186449 2024-11-10T19:16:29,611 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 674d18e4edb14b80af3ae7b7797f1785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731266187174 2024-11-10T19:16:29,612 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting dd69442c4b694be58c27263e177e2b2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731266189355 2024-11-10T19:16:29,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/8fd6395497d149d2af9f2b28f7ceb816 is 50, key is test_row_0/B:col10/1731266189392/Put/seqid=0 2024-11-10T19:16:29,629 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:29,630 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/9bf75ea8dd2c44e4b48759a073e402e1 is 50, key is test_row_0/C:col10/1731266189356/Put/seqid=0 2024-11-10T19:16:29,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741909_1085 (size=12301) 2024-11-10T19:16:29,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741910_1086 (size=13085) 2024-11-10T19:16:29,681 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/9bf75ea8dd2c44e4b48759a073e402e1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9bf75ea8dd2c44e4b48759a073e402e1 2024-11-10T19:16:29,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266249682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266249682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,692 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 9bf75ea8dd2c44e4b48759a073e402e1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:29,693 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:29,693 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=12, startTime=1731266189488; duration=0sec 2024-11-10T19:16:29,693 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:29,693 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266249691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266249691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266249692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T19:16:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T19:16:29,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T19:16:29,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:29,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:29,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:29,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:29,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266249891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266249902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266249902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266249902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:29,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266249891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/8fd6395497d149d2af9f2b28f7ceb816 2024-11-10T19:16:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T19:16:30,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T19:16:30,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:30,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:30,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:30,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:30,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:30,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/d020d502a5ac49ea979553274414987e is 50, key is test_row_0/C:col10/1731266189392/Put/seqid=0 2024-11-10T19:16:30,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741911_1087 (size=12301) 2024-11-10T19:16:30,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/d020d502a5ac49ea979553274414987e 2024-11-10T19:16:30,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/4bf770ca5c724cecb95ca32f47713434 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4bf770ca5c724cecb95ca32f47713434 2024-11-10T19:16:30,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4bf770ca5c724cecb95ca32f47713434, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T19:16:30,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/8fd6395497d149d2af9f2b28f7ceb816 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/8fd6395497d149d2af9f2b28f7ceb816 2024-11-10T19:16:30,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/8fd6395497d149d2af9f2b28f7ceb816, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T19:16:30,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/d020d502a5ac49ea979553274414987e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d020d502a5ac49ea979553274414987e 2024-11-10T19:16:30,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d020d502a5ac49ea979553274414987e, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T19:16:30,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~202.13 KB/206976, currentSize=107.34 KB/109920 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 629ms, sequenceid=319, compaction requested=false 2024-11-10T19:16:30,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:30,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T19:16:30,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:30,199 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T19:16:30,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:30,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:30,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:30,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:30,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:30,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/d12259a225f34ed7b059d7597264f42d is 50, key is test_row_0/A:col10/1731266189569/Put/seqid=0 2024-11-10T19:16:30,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741912_1088 (size=12301) 2024-11-10T19:16:30,255 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/d12259a225f34ed7b059d7597264f42d 2024-11-10T19:16:30,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266250248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266250252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266250255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266250265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266250265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/de5164a4639e49afbf1f9128cc96a318 is 50, key is test_row_0/B:col10/1731266189569/Put/seqid=0 2024-11-10T19:16:30,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741913_1089 (size=12301) 2024-11-10T19:16:30,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266250372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266250372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266250372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266250381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266250391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T19:16:30,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266250581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266250581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266250581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266250591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266250601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,697 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/de5164a4639e49afbf1f9128cc96a318 2024-11-10T19:16:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/e6ec3012070240c6bcce682ef7ce6184 is 50, key is test_row_0/C:col10/1731266189569/Put/seqid=0 2024-11-10T19:16:30,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741914_1090 (size=12301) 2024-11-10T19:16:30,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266250892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266250892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266250892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266250902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:30,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266250911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,137 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/e6ec3012070240c6bcce682ef7ce6184 2024-11-10T19:16:31,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/d12259a225f34ed7b059d7597264f42d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d12259a225f34ed7b059d7597264f42d 2024-11-10T19:16:31,157 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d12259a225f34ed7b059d7597264f42d, entries=150, sequenceid=341, filesize=12.0 K 2024-11-10T19:16:31,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/de5164a4639e49afbf1f9128cc96a318 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/de5164a4639e49afbf1f9128cc96a318 2024-11-10T19:16:31,169 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/de5164a4639e49afbf1f9128cc96a318, entries=150, sequenceid=341, filesize=12.0 K 2024-11-10T19:16:31,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/e6ec3012070240c6bcce682ef7ce6184 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e6ec3012070240c6bcce682ef7ce6184 2024-11-10T19:16:31,183 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e6ec3012070240c6bcce682ef7ce6184, entries=150, sequenceid=341, filesize=12.0 K 2024-11-10T19:16:31,184 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 985ms, sequenceid=341, compaction requested=true 2024-11-10T19:16:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:31,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:31,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T19:16:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T19:16:31,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T19:16:31,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-11-10T19:16:31,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7860 sec 2024-11-10T19:16:31,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:31,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:31,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:31,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:31,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:31,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:31,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:31,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a2d7c5bd484d4c6bbd47021d65d13ed9 is 50, key is test_row_0/A:col10/1731266191405/Put/seqid=0 2024-11-10T19:16:31,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741915_1091 (size=12301) 2024-11-10T19:16:31,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a2d7c5bd484d4c6bbd47021d65d13ed9 2024-11-10T19:16:31,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/aad0f5cd74cb445298da494747d2a1e2 is 50, key is test_row_0/B:col10/1731266191405/Put/seqid=0 2024-11-10T19:16:31,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266251491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266251491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266251509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266251513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266251523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741916_1092 (size=12301) 2024-11-10T19:16:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T19:16:31,584 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:31,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T19:16:31,594 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:31,596 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:31,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T19:16:31,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266251633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266251641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266251642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266251642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266251642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T19:16:31,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:31,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:31,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:31,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:31,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266251845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266251852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266251866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266251866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266251868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T19:16:31,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:31,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:31,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:31,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:31,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:31,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:31,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:31,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/aad0f5cd74cb445298da494747d2a1e2 2024-11-10T19:16:32,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/e3a4b15e0da64b2f998c6e5073ebf4be is 50, key is test_row_0/C:col10/1731266191405/Put/seqid=0 2024-11-10T19:16:32,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:32,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:32,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741917_1093 (size=12301) 2024-11-10T19:16:32,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266252165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266252177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266252184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266252191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266252191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T19:16:32,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:32,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:32,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:32,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:32,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/e3a4b15e0da64b2f998c6e5073ebf4be 2024-11-10T19:16:32,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/a2d7c5bd484d4c6bbd47021d65d13ed9 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a2d7c5bd484d4c6bbd47021d65d13ed9 2024-11-10T19:16:32,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a2d7c5bd484d4c6bbd47021d65d13ed9, entries=150, sequenceid=357, filesize=12.0 K 2024-11-10T19:16:32,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/aad0f5cd74cb445298da494747d2a1e2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/aad0f5cd74cb445298da494747d2a1e2 2024-11-10T19:16:32,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/aad0f5cd74cb445298da494747d2a1e2, entries=150, sequenceid=357, filesize=12.0 K 2024-11-10T19:16:32,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/e3a4b15e0da64b2f998c6e5073ebf4be as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e3a4b15e0da64b2f998c6e5073ebf4be 2024-11-10T19:16:32,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:32,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:32,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e3a4b15e0da64b2f998c6e5073ebf4be, entries=150, sequenceid=357, filesize=12.0 K 2024-11-10T19:16:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 1142ms, sequenceid=357, compaction requested=true 2024-11-10T19:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:32,559 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:32,559 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:32,561 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:16:32,562 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:32,562 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,562 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5d3c6f23d49e408181db585b74b0c3f4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4bf770ca5c724cecb95ca32f47713434, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d12259a225f34ed7b059d7597264f42d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a2d7c5bd484d4c6bbd47021d65d13ed9] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=48.8 K 2024-11-10T19:16:32,563 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3c6f23d49e408181db585b74b0c3f4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731266189355 2024-11-10T19:16:32,563 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:16:32,563 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf770ca5c724cecb95ca32f47713434, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731266189392 2024-11-10T19:16:32,563 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:32,563 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,563 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/35b760ddc09e45e182135a11c4daf166, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/8fd6395497d149d2af9f2b28f7ceb816, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/de5164a4639e49afbf1f9128cc96a318, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/aad0f5cd74cb445298da494747d2a1e2] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=48.8 K 2024-11-10T19:16:32,564 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting d12259a225f34ed7b059d7597264f42d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731266189542 2024-11-10T19:16:32,567 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b760ddc09e45e182135a11c4daf166, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731266189355 2024-11-10T19:16:32,568 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d7c5bd484d4c6bbd47021d65d13ed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731266190254 2024-11-10T19:16:32,572 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd6395497d149d2af9f2b28f7ceb816, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731266189392 2024-11-10T19:16:32,573 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting de5164a4639e49afbf1f9128cc96a318, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731266189542 2024-11-10T19:16:32,574 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting aad0f5cd74cb445298da494747d2a1e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731266190254 2024-11-10T19:16:32,591 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:32,591 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/3292101355b943ce9aaa3530ee10afff is 50, key is test_row_0/B:col10/1731266191405/Put/seqid=0 2024-11-10T19:16:32,603 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:32,604 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/9d1f9287382846b29e840aadde38086e is 50, key is test_row_0/A:col10/1731266191405/Put/seqid=0 2024-11-10T19:16:32,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741918_1094 (size=13221) 2024-11-10T19:16:32,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741919_1095 (size=13221) 2024-11-10T19:16:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:32,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:32,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:32,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:32,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:32,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/15599f8d9ab44c9d8680d839cbff8502 is 50, key is test_row_0/A:col10/1731266192707/Put/seqid=0 2024-11-10T19:16:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T19:16:32,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266252749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266252749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266252749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266252753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266252756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741920_1096 (size=14741) 2024-11-10T19:16:32,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/15599f8d9ab44c9d8680d839cbff8502 2024-11-10T19:16:32,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/d30aef4966b84fdfbc01c2f2b1e2ba1b is 50, key is test_row_0/B:col10/1731266192707/Put/seqid=0 2024-11-10T19:16:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266252862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:32,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:32,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:32,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266252871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741921_1097 (size=12301) 2024-11-10T19:16:32,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266252872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266252872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/d30aef4966b84fdfbc01c2f2b1e2ba1b 2024-11-10T19:16:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266252872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:32,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/dc2ebe7953d4461ba3065b8e81a8099a is 50, key is test_row_0/C:col10/1731266192707/Put/seqid=0 2024-11-10T19:16:33,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741922_1098 (size=12301) 2024-11-10T19:16:33,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:33,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:33,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/dc2ebe7953d4461ba3065b8e81a8099a 2024-11-10T19:16:33,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/15599f8d9ab44c9d8680d839cbff8502 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/15599f8d9ab44c9d8680d839cbff8502 2024-11-10T19:16:33,075 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/9d1f9287382846b29e840aadde38086e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9d1f9287382846b29e840aadde38086e 2024-11-10T19:16:33,076 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/3292101355b943ce9aaa3530ee10afff as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3292101355b943ce9aaa3530ee10afff 2024-11-10T19:16:33,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/15599f8d9ab44c9d8680d839cbff8502, entries=200, sequenceid=374, filesize=14.4 K 2024-11-10T19:16:33,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/d30aef4966b84fdfbc01c2f2b1e2ba1b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d30aef4966b84fdfbc01c2f2b1e2ba1b 2024-11-10T19:16:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266253081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266253093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,103 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 3292101355b943ce9aaa3530ee10afff(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:33,103 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:33,103 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=12, startTime=1731266192559; duration=0sec 2024-11-10T19:16:33,103 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:33,103 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:33,104 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:16:33,106 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 9d1f9287382846b29e840aadde38086e(size=12.9 K), total size for store is 27.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:33,106 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:33,107 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=12, startTime=1731266192559; duration=0sec 2024-11-10T19:16:33,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,107 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:33,108 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266253101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266253102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d30aef4966b84fdfbc01c2f2b1e2ba1b, entries=150, sequenceid=374, filesize=12.0 K 2024-11-10T19:16:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,112 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:16:33,112 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:33,112 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,112 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9bf75ea8dd2c44e4b48759a073e402e1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d020d502a5ac49ea979553274414987e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e6ec3012070240c6bcce682ef7ce6184, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e3a4b15e0da64b2f998c6e5073ebf4be] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=48.8 K 2024-11-10T19:16:33,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/dc2ebe7953d4461ba3065b8e81a8099a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dc2ebe7953d4461ba3065b8e81a8099a 2024-11-10T19:16:33,113 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf75ea8dd2c44e4b48759a073e402e1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731266189355 2024-11-10T19:16:33,114 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting d020d502a5ac49ea979553274414987e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731266189392 2024-11-10T19:16:33,115 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ec3012070240c6bcce682ef7ce6184, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731266189542 2024-11-10T19:16:33,115 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a4b15e0da64b2f998c6e5073ebf4be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731266190254 2024-11-10T19:16:33,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dc2ebe7953d4461ba3065b8e81a8099a, entries=150, sequenceid=374, filesize=12.0 K 2024-11-10T19:16:33,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 443ms, sequenceid=374, compaction requested=false 2024-11-10T19:16:33,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:33,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:33,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:33,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:33,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:33,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:33,186 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:33,187 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/27ff56686513418abd48450d14fa597c is 50, key is test_row_0/C:col10/1731266191405/Put/seqid=0 2024-11-10T19:16:33,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/10cfd09c5e5048d798ac6688422c7c98 is 50, key is test_row_0/A:col10/1731266193167/Put/seqid=0 2024-11-10T19:16:33,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:33,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:33,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741923_1099 (size=13221) 2024-11-10T19:16:33,260 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/27ff56686513418abd48450d14fa597c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/27ff56686513418abd48450d14fa597c 2024-11-10T19:16:33,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741924_1100 (size=14741) 2024-11-10T19:16:33,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/10cfd09c5e5048d798ac6688422c7c98 2024-11-10T19:16:33,282 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 27ff56686513418abd48450d14fa597c(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:33,282 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:33,282 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=12, startTime=1731266192559; duration=0sec 2024-11-10T19:16:33,282 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:33,282 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:33,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/c9491f20d2c549dfaff9d509adfe4361 is 50, key is test_row_0/B:col10/1731266193167/Put/seqid=0 2024-11-10T19:16:33,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741925_1101 (size=12301) 2024-11-10T19:16:33,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:33,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:33,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266253402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266253405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266253405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266253414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266253425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266253524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:33,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:33,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266253733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T19:16:33,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/c9491f20d2c549dfaff9d509adfe4361 2024-11-10T19:16:33,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/bfce815c92d14bf1810129e99a65593f is 50, key is test_row_0/C:col10/1731266193167/Put/seqid=0 2024-11-10T19:16:33,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741926_1102 (size=12301) 2024-11-10T19:16:33,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/bfce815c92d14bf1810129e99a65593f 2024-11-10T19:16:33,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/10cfd09c5e5048d798ac6688422c7c98 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/10cfd09c5e5048d798ac6688422c7c98 2024-11-10T19:16:33,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:33,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:33,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/10cfd09c5e5048d798ac6688422c7c98, entries=200, sequenceid=392, filesize=14.4 K 2024-11-10T19:16:33,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/c9491f20d2c549dfaff9d509adfe4361 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/c9491f20d2c549dfaff9d509adfe4361 2024-11-10T19:16:33,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/c9491f20d2c549dfaff9d509adfe4361, entries=150, sequenceid=392, filesize=12.0 K 2024-11-10T19:16:33,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/bfce815c92d14bf1810129e99a65593f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/bfce815c92d14bf1810129e99a65593f 2024-11-10T19:16:33,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/bfce815c92d14bf1810129e99a65593f, entries=150, sequenceid=392, filesize=12.0 K 2024-11-10T19:16:33,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 752ms, sequenceid=392, compaction requested=true 2024-11-10T19:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:33,924 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:33,924 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:33,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:33,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:33,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:33,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:33,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:33,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:33,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:33,928 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:33,928 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:33,928 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,928 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9d1f9287382846b29e840aadde38086e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/15599f8d9ab44c9d8680d839cbff8502, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/10cfd09c5e5048d798ac6688422c7c98] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=41.7 K 2024-11-10T19:16:33,929 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:33,929 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:33,929 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:33,929 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3292101355b943ce9aaa3530ee10afff, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d30aef4966b84fdfbc01c2f2b1e2ba1b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/c9491f20d2c549dfaff9d509adfe4361] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=36.9 K 2024-11-10T19:16:33,936 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d1f9287382846b29e840aadde38086e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731266190254 2024-11-10T19:16:33,936 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3292101355b943ce9aaa3530ee10afff, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731266190254 2024-11-10T19:16:33,937 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting d30aef4966b84fdfbc01c2f2b1e2ba1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731266191476 2024-11-10T19:16:33,937 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15599f8d9ab44c9d8680d839cbff8502, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731266191476 2024-11-10T19:16:33,937 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c9491f20d2c549dfaff9d509adfe4361, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731266192745 2024-11-10T19:16:33,938 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10cfd09c5e5048d798ac6688422c7c98, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731266192745 2024-11-10T19:16:33,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/d2d5e602062b4bf8a2517d35dccd8200 is 50, key is test_row_0/A:col10/1731266193925/Put/seqid=0 2024-11-10T19:16:33,957 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:33,957 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/e870382b858947ec80cee78a80caf224 is 50, key is test_row_0/B:col10/1731266193167/Put/seqid=0 2024-11-10T19:16:33,964 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#92 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:33,965 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/20a8dd3c23bf4dcfa6955e3e7f1a218e is 50, key is test_row_0/A:col10/1731266193167/Put/seqid=0 2024-11-10T19:16:33,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266253966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266253966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266253967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266253968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741927_1103 (size=14741) 2024-11-10T19:16:34,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:34,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:34,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741928_1104 (size=13323) 2024-11-10T19:16:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741929_1105 (size=13323) 2024-11-10T19:16:34,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266254044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,056 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/20a8dd3c23bf4dcfa6955e3e7f1a218e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/20a8dd3c23bf4dcfa6955e3e7f1a218e 2024-11-10T19:16:34,072 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 20a8dd3c23bf4dcfa6955e3e7f1a218e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:34,072 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:34,072 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=13, startTime=1731266193924; duration=0sec 2024-11-10T19:16:34,073 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:34,073 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:34,073 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:34,076 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:34,077 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:34,077 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,077 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/27ff56686513418abd48450d14fa597c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dc2ebe7953d4461ba3065b8e81a8099a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/bfce815c92d14bf1810129e99a65593f] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=36.9 K 2024-11-10T19:16:34,078 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27ff56686513418abd48450d14fa597c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731266190254 2024-11-10T19:16:34,078 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc2ebe7953d4461ba3065b8e81a8099a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731266191476 2024-11-10T19:16:34,078 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfce815c92d14bf1810129e99a65593f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731266192745 2024-11-10T19:16:34,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266254082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266254084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,098 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:34,099 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/5f18c29a5a1f4a57aa95e1368c094503 is 50, key is test_row_0/C:col10/1731266193167/Put/seqid=0 2024-11-10T19:16:34,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266254093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266254093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741930_1106 (size=13323) 2024-11-10T19:16:34,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,204 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/5f18c29a5a1f4a57aa95e1368c094503 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5f18c29a5a1f4a57aa95e1368c094503 2024-11-10T19:16:34,231 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 5f18c29a5a1f4a57aa95e1368c094503(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:34,231 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:34,231 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=13, startTime=1731266193926; duration=0sec 2024-11-10T19:16:34,231 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:34,231 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:34,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266254291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266254292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266254311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266254311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:34,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:34,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/d2d5e602062b4bf8a2517d35dccd8200 2024-11-10T19:16:34,416 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/e870382b858947ec80cee78a80caf224 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/e870382b858947ec80cee78a80caf224 2024-11-10T19:16:34,428 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into e870382b858947ec80cee78a80caf224(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:34,428 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:34,428 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=13, startTime=1731266193924; duration=0sec 2024-11-10T19:16:34,428 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:34,428 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:34,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/2eb40c8f7423427889bbd17f18cf7350 is 50, key is test_row_0/B:col10/1731266193925/Put/seqid=0 2024-11-10T19:16:34,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741931_1107 (size=12301) 2024-11-10T19:16:34,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/2eb40c8f7423427889bbd17f18cf7350 2024-11-10T19:16:34,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/ed23da271e3b422aae246d7563444cd0 is 50, key is test_row_0/C:col10/1731266193925/Put/seqid=0 2024-11-10T19:16:34,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:34,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741932_1108 (size=12301) 2024-11-10T19:16:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/ed23da271e3b422aae246d7563444cd0 2024-11-10T19:16:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/d2d5e602062b4bf8a2517d35dccd8200 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d2d5e602062b4bf8a2517d35dccd8200 2024-11-10T19:16:34,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d2d5e602062b4bf8a2517d35dccd8200, entries=200, sequenceid=410, filesize=14.4 K 2024-11-10T19:16:34,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/2eb40c8f7423427889bbd17f18cf7350 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb40c8f7423427889bbd17f18cf7350 2024-11-10T19:16:34,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb40c8f7423427889bbd17f18cf7350, entries=150, sequenceid=410, filesize=12.0 K 2024-11-10T19:16:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/ed23da271e3b422aae246d7563444cd0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/ed23da271e3b422aae246d7563444cd0 2024-11-10T19:16:34,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266254551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/ed23da271e3b422aae246d7563444cd0, entries=150, sequenceid=410, filesize=12.0 K 2024-11-10T19:16:34,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 637ms, sequenceid=410, compaction requested=false 2024-11-10T19:16:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:34,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:34,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:34,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:34,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:34,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:34,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/e459877a06fc4931ac46523d7843a91b is 50, key is test_row_0/A:col10/1731266194603/Put/seqid=0 2024-11-10T19:16:34,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266254637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266254639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266254643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266254653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266254806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741933_1109 (size=17181) 2024-11-10T19:16:34,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/e459877a06fc4931ac46523d7843a91b 2024-11-10T19:16:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266254808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266254808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266254808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:34,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:34,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/783096819712435da33dcaf1f2a72fc3 is 50, key is test_row_0/B:col10/1731266194603/Put/seqid=0 2024-11-10T19:16:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741934_1110 (size=12301) 2024-11-10T19:16:34,864 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-10T19:16:34,864 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-10T19:16:34,864 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:16:34,864 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:16:34,868 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:34,868 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:34,869 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:34,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:34,869 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:34,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:34,872 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-10T19:16:34,872 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:16:34,872 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:34,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:34,872 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:34,879 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-10T19:16:34,879 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:16:34,879 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:34,880 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:34,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:34,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:34,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:34,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:34,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266255022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:35,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266255023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:35,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:35,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266255025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:35,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266255025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:35,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/783096819712435da33dcaf1f2a72fc3 2024-11-10T19:16:35,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/09784408cefa4e2ca654a200c648d9de is 50, key is test_row_0/C:col10/1731266194603/Put/seqid=0 2024-11-10T19:16:35,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741935_1111 (size=12301) 2024-11-10T19:16:35,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:35,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:35,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39938 deadline: 1731266255331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39986 deadline: 1731266255331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39908 deadline: 1731266255331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:35,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:35,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39910 deadline: 1731266255342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:35,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39922 deadline: 1731266255571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:35,572 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in 4f5e56bdcde85bbc88e3b9e5355a12b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:16:35,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:35,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/09784408cefa4e2ca654a200c648d9de 2024-11-10T19:16:35,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/e459877a06fc4931ac46523d7843a91b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e459877a06fc4931ac46523d7843a91b 2024-11-10T19:16:35,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e459877a06fc4931ac46523d7843a91b, entries=250, sequenceid=430, filesize=16.8 K 2024-11-10T19:16:35,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/783096819712435da33dcaf1f2a72fc3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/783096819712435da33dcaf1f2a72fc3 2024-11-10T19:16:35,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/783096819712435da33dcaf1f2a72fc3, entries=150, sequenceid=430, filesize=12.0 K 2024-11-10T19:16:35,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/09784408cefa4e2ca654a200c648d9de as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/09784408cefa4e2ca654a200c648d9de 2024-11-10T19:16:35,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/09784408cefa4e2ca654a200c648d9de, entries=150, sequenceid=430, filesize=12.0 K 2024-11-10T19:16:35,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 1072ms, sequenceid=430, compaction requested=true 2024-11-10T19:16:35,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:35,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:35,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:35,676 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:35,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:35,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:35,677 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:35,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f5e56bdcde85bbc88e3b9e5355a12b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:35,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:35,678 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:35,678 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:35,678 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/B is initiating minor compaction (all files) 2024-11-10T19:16:35,678 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/A is initiating minor compaction (all files) 2024-11-10T19:16:35,678 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/B in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,678 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/A in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,678 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/e870382b858947ec80cee78a80caf224, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb40c8f7423427889bbd17f18cf7350, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/783096819712435da33dcaf1f2a72fc3] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=37.0 K 2024-11-10T19:16:35,678 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/20a8dd3c23bf4dcfa6955e3e7f1a218e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d2d5e602062b4bf8a2517d35dccd8200, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e459877a06fc4931ac46523d7843a91b] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=44.2 K 2024-11-10T19:16:35,678 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting e870382b858947ec80cee78a80caf224, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731266192745 2024-11-10T19:16:35,678 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20a8dd3c23bf4dcfa6955e3e7f1a218e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731266192745 2024-11-10T19:16:35,679 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eb40c8f7423427889bbd17f18cf7350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731266193330 2024-11-10T19:16:35,679 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2d5e602062b4bf8a2517d35dccd8200, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731266193330 2024-11-10T19:16:35,679 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 783096819712435da33dcaf1f2a72fc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731266193962 2024-11-10T19:16:35,679 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting e459877a06fc4931ac46523d7843a91b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731266193953 2024-11-10T19:16:35,699 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#B#compaction#99 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:35,700 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/af3ef249ac9140a79d5c2d1c6bbbd0bf is 50, key is test_row_0/B:col10/1731266194603/Put/seqid=0 2024-11-10T19:16:35,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741936_1112 (size=13425) 2024-11-10T19:16:35,719 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/af3ef249ac9140a79d5c2d1c6bbbd0bf as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/af3ef249ac9140a79d5c2d1c6bbbd0bf 2024-11-10T19:16:35,725 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#A#compaction#100 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:35,726 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/50eaa165c306440d857872bca75b4ac3 is 50, key is test_row_0/A:col10/1731266194603/Put/seqid=0 2024-11-10T19:16:35,726 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/B of 4f5e56bdcde85bbc88e3b9e5355a12b5 into af3ef249ac9140a79d5c2d1c6bbbd0bf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:35,727 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:35,727 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/B, priority=13, startTime=1731266195676; duration=0sec 2024-11-10T19:16:35,727 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:35,727 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:B 2024-11-10T19:16:35,727 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:35,729 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:35,729 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4f5e56bdcde85bbc88e3b9e5355a12b5/C is initiating minor compaction (all files) 2024-11-10T19:16:35,729 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f5e56bdcde85bbc88e3b9e5355a12b5/C in TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,729 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5f18c29a5a1f4a57aa95e1368c094503, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/ed23da271e3b422aae246d7563444cd0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/09784408cefa4e2ca654a200c648d9de] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp, totalSize=37.0 K 2024-11-10T19:16:35,730 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f18c29a5a1f4a57aa95e1368c094503, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731266192745 2024-11-10T19:16:35,730 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ed23da271e3b422aae246d7563444cd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731266193330 2024-11-10T19:16:35,730 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 09784408cefa4e2ca654a200c648d9de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731266193962 2024-11-10T19:16:35,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T19:16:35,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:35,737 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T19:16:35,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:35,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:35,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741937_1113 (size=13425) 2024-11-10T19:16:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/e20c69f5e0c146c883947f53c594187f is 50, key is test_row_0/A:col10/1731266194641/Put/seqid=0 2024-11-10T19:16:35,765 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f5e56bdcde85bbc88e3b9e5355a12b5#C#compaction#102 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:35,766 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/8d4cc20210bb4289a8300a1464abdc15 is 50, key is test_row_0/C:col10/1731266194603/Put/seqid=0 2024-11-10T19:16:35,768 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/50eaa165c306440d857872bca75b4ac3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/50eaa165c306440d857872bca75b4ac3 2024-11-10T19:16:35,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741938_1114 (size=12301) 2024-11-10T19:16:35,776 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/e20c69f5e0c146c883947f53c594187f 2024-11-10T19:16:35,781 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/A of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 50eaa165c306440d857872bca75b4ac3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:35,781 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:35,782 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/A, priority=13, startTime=1731266195676; duration=0sec 2024-11-10T19:16:35,783 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:35,783 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:A 2024-11-10T19:16:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T19:16:35,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741939_1115 (size=13425) 2024-11-10T19:16:35,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/d84bc42bbe224a82a9f8d16a329b2310 is 50, key is test_row_0/B:col10/1731266194641/Put/seqid=0 2024-11-10T19:16:35,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741940_1116 (size=12301) 2024-11-10T19:16:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:35,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. as already flushing 2024-11-10T19:16:35,844 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-10T19:16:35,844 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:16:35,844 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:35,844 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-10T19:16:35,844 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-10T19:16:35,844 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:16:35,844 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:16:35,844 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:35,844 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:35,844 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:35,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:35,844 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:35,844 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:35,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:35,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:35,853 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-10T19:16:35,853 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:16:35,853 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:35,853 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:35,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:35,897 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T19:16:36,205 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/8d4cc20210bb4289a8300a1464abdc15 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/8d4cc20210bb4289a8300a1464abdc15 2024-11-10T19:16:36,209 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/d84bc42bbe224a82a9f8d16a329b2310 2024-11-10T19:16:36,215 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f5e56bdcde85bbc88e3b9e5355a12b5/C of 4f5e56bdcde85bbc88e3b9e5355a12b5 into 8d4cc20210bb4289a8300a1464abdc15(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:36,215 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:36,215 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., storeName=4f5e56bdcde85bbc88e3b9e5355a12b5/C, priority=13, startTime=1731266195677; duration=0sec 2024-11-10T19:16:36,215 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:36,215 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f5e56bdcde85bbc88e3b9e5355a12b5:C 2024-11-10T19:16:36,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/3947e30e52964116ab111eb1d1fb7158 is 50, key is test_row_0/C:col10/1731266194641/Put/seqid=0 2024-11-10T19:16:36,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741941_1117 (size=12301) 2024-11-10T19:16:36,626 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/3947e30e52964116ab111eb1d1fb7158 2024-11-10T19:16:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/e20c69f5e0c146c883947f53c594187f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e20c69f5e0c146c883947f53c594187f 2024-11-10T19:16:36,636 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e20c69f5e0c146c883947f53c594187f, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T19:16:36,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/d84bc42bbe224a82a9f8d16a329b2310 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d84bc42bbe224a82a9f8d16a329b2310 2024-11-10T19:16:36,643 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d84bc42bbe224a82a9f8d16a329b2310, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T19:16:36,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/3947e30e52964116ab111eb1d1fb7158 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3947e30e52964116ab111eb1d1fb7158 2024-11-10T19:16:36,650 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3947e30e52964116ab111eb1d1fb7158, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T19:16:36,651 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 914ms, sequenceid=446, compaction requested=false 2024-11-10T19:16:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: 2024-11-10T19:16:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T19:16:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T19:16:36,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T19:16:36,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.0570 sec 2024-11-10T19:16:36,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 5.0680 sec 2024-11-10T19:16:37,583 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-10T19:16:37,583 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:16:37,583 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:37,583 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:37,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T19:16:39,831 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:39,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 166 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2169 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2225 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1025 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3072 rows 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1018 2024-11-10T19:16:39,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-11-10T19:16:39,832 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:16:39,832 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:16:39,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:39,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:39,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:16:39,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T19:16:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:39,844 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266199844"}]},"ts":"1731266199844"} 2024-11-10T19:16:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T19:16:39,846 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T19:16:39,846 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T19:16:39,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T19:16:39,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5, UNASSIGN}] 2024-11-10T19:16:39,855 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5, UNASSIGN 2024-11-10T19:16:39,857 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=4f5e56bdcde85bbc88e3b9e5355a12b5, regionState=CLOSING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:39,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5, UNASSIGN because future has completed 2024-11-10T19:16:39,860 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T19:16:39,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:16:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T19:16:40,018 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] handler.UnassignRegionHandler(122): Close 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:40,019 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T19:16:40,020 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegion(1722): Closing 4f5e56bdcde85bbc88e3b9e5355a12b5, disabling compactions & flushes 2024-11-10T19:16:40,020 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:40,020 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:40,020 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. after waiting 0 ms 2024-11-10T19:16:40,020 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:40,020 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegion(2902): Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T19:16:40,020 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=A 2024-11-10T19:16:40,021 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:40,021 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=B 2024-11-10T19:16:40,021 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:40,021 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4f5e56bdcde85bbc88e3b9e5355a12b5, store=C 2024-11-10T19:16:40,021 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:40,026 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/03faca5477094f5bbe8bb92c0df54bdd is 50, key is test_row_0/A:col10/1731266195842/Put/seqid=0 2024-11-10T19:16:40,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741942_1118 (size=12301) 2024-11-10T19:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T19:16:40,432 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/03faca5477094f5bbe8bb92c0df54bdd 2024-11-10T19:16:40,446 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/993f15e3883a4529b5f0cc1f55f705b6 is 50, key is test_row_0/B:col10/1731266195842/Put/seqid=0 2024-11-10T19:16:40,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741943_1119 (size=12301) 2024-11-10T19:16:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T19:16:40,851 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/993f15e3883a4529b5f0cc1f55f705b6 2024-11-10T19:16:40,860 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/3118803e6ae249d193bee5ddecf8697d is 50, key is test_row_0/C:col10/1731266195842/Put/seqid=0 2024-11-10T19:16:40,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741944_1120 (size=12301) 2024-11-10T19:16:40,867 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/3118803e6ae249d193bee5ddecf8697d 2024-11-10T19:16:40,874 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/A/03faca5477094f5bbe8bb92c0df54bdd as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/03faca5477094f5bbe8bb92c0df54bdd 2024-11-10T19:16:40,880 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/03faca5477094f5bbe8bb92c0df54bdd, entries=150, sequenceid=456, filesize=12.0 K 2024-11-10T19:16:40,881 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/B/993f15e3883a4529b5f0cc1f55f705b6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/993f15e3883a4529b5f0cc1f55f705b6 2024-11-10T19:16:40,887 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/993f15e3883a4529b5f0cc1f55f705b6, entries=150, sequenceid=456, filesize=12.0 K 2024-11-10T19:16:40,889 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/.tmp/C/3118803e6ae249d193bee5ddecf8697d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3118803e6ae249d193bee5ddecf8697d 2024-11-10T19:16:40,895 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3118803e6ae249d193bee5ddecf8697d, entries=150, sequenceid=456, filesize=12.0 K 2024-11-10T19:16:40,896 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 876ms, sequenceid=456, compaction requested=true 2024-11-10T19:16:40,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c731462d6b6d47bf8d2d9798518a6bc7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5b5baf27efe24231a8417781b236d422, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/7513d82d7b5342adaf2d64e7cbd131c5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a47d329c598f433c859c7f1f57e0a4c6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/2ca277d8bce94fbeb6d07b7df52d4f71, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/6b209552aae9424bb1611415b7a24849, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a0db9049a52f4e9c968ff00cc115c7e5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8d4edd23a6eb4f5a95b9707b599db202, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/42447c89be0f437183302e2ac366dac7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4ce7744d3611480f8fdbe440d1a78e01, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/f310d8e122b240fbab9ffe173fa23a17, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a5414d69e6994321834748a709eb549d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5c637dce15d349b29daf37fa2fc7005e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0e85da0a02364c419c6421fe6ea10211, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/14192518050b4ba4a7cd3ec93f112042, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c2e1f77ed4734126bd3079bd34cb9d38, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d0a1c385c00941f2acdc108969742bd1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/861a4e40ffe04a6e910c51c1cab9c722, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8098ba351ac24036a66573ed1eaf1ee4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/df10501395b54a11871ac52cffaec306, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0bd5b55125674f339416a61adb3546cf, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9ce110cb88b0422c9af762c350d0d1c5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/bcb6289f1bf145c78a452547df114e5e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5d3c6f23d49e408181db585b74b0c3f4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4bf770ca5c724cecb95ca32f47713434, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d12259a225f34ed7b059d7597264f42d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9d1f9287382846b29e840aadde38086e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a2d7c5bd484d4c6bbd47021d65d13ed9, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/15599f8d9ab44c9d8680d839cbff8502, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/10cfd09c5e5048d798ac6688422c7c98, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/20a8dd3c23bf4dcfa6955e3e7f1a218e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d2d5e602062b4bf8a2517d35dccd8200, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e459877a06fc4931ac46523d7843a91b] to archive 2024-11-10T19:16:40,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:16:40,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c731462d6b6d47bf8d2d9798518a6bc7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c731462d6b6d47bf8d2d9798518a6bc7 2024-11-10T19:16:40,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5b5baf27efe24231a8417781b236d422 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5b5baf27efe24231a8417781b236d422 2024-11-10T19:16:40,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/7513d82d7b5342adaf2d64e7cbd131c5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/7513d82d7b5342adaf2d64e7cbd131c5 2024-11-10T19:16:40,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a47d329c598f433c859c7f1f57e0a4c6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a47d329c598f433c859c7f1f57e0a4c6 2024-11-10T19:16:40,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/2ca277d8bce94fbeb6d07b7df52d4f71 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/2ca277d8bce94fbeb6d07b7df52d4f71 2024-11-10T19:16:40,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/6b209552aae9424bb1611415b7a24849 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/6b209552aae9424bb1611415b7a24849 2024-11-10T19:16:40,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a0db9049a52f4e9c968ff00cc115c7e5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a0db9049a52f4e9c968ff00cc115c7e5 2024-11-10T19:16:40,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8d4edd23a6eb4f5a95b9707b599db202 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8d4edd23a6eb4f5a95b9707b599db202 2024-11-10T19:16:40,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/42447c89be0f437183302e2ac366dac7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/42447c89be0f437183302e2ac366dac7 2024-11-10T19:16:40,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4ce7744d3611480f8fdbe440d1a78e01 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4ce7744d3611480f8fdbe440d1a78e01 2024-11-10T19:16:40,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/f310d8e122b240fbab9ffe173fa23a17 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/f310d8e122b240fbab9ffe173fa23a17 2024-11-10T19:16:40,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a5414d69e6994321834748a709eb549d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a5414d69e6994321834748a709eb549d 2024-11-10T19:16:40,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5c637dce15d349b29daf37fa2fc7005e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5c637dce15d349b29daf37fa2fc7005e 2024-11-10T19:16:40,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0e85da0a02364c419c6421fe6ea10211 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0e85da0a02364c419c6421fe6ea10211 2024-11-10T19:16:40,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/14192518050b4ba4a7cd3ec93f112042 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/14192518050b4ba4a7cd3ec93f112042 2024-11-10T19:16:40,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c2e1f77ed4734126bd3079bd34cb9d38 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/c2e1f77ed4734126bd3079bd34cb9d38 2024-11-10T19:16:40,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d0a1c385c00941f2acdc108969742bd1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d0a1c385c00941f2acdc108969742bd1 2024-11-10T19:16:40,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/861a4e40ffe04a6e910c51c1cab9c722 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/861a4e40ffe04a6e910c51c1cab9c722 2024-11-10T19:16:40,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8098ba351ac24036a66573ed1eaf1ee4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/8098ba351ac24036a66573ed1eaf1ee4 2024-11-10T19:16:40,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/df10501395b54a11871ac52cffaec306 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/df10501395b54a11871ac52cffaec306 2024-11-10T19:16:40,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0bd5b55125674f339416a61adb3546cf to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/0bd5b55125674f339416a61adb3546cf 2024-11-10T19:16:40,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9ce110cb88b0422c9af762c350d0d1c5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9ce110cb88b0422c9af762c350d0d1c5 2024-11-10T19:16:40,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/bcb6289f1bf145c78a452547df114e5e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/bcb6289f1bf145c78a452547df114e5e 2024-11-10T19:16:40,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5d3c6f23d49e408181db585b74b0c3f4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/5d3c6f23d49e408181db585b74b0c3f4 2024-11-10T19:16:40,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4bf770ca5c724cecb95ca32f47713434 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/4bf770ca5c724cecb95ca32f47713434 2024-11-10T19:16:40,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d12259a225f34ed7b059d7597264f42d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d12259a225f34ed7b059d7597264f42d 2024-11-10T19:16:40,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9d1f9287382846b29e840aadde38086e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/9d1f9287382846b29e840aadde38086e 2024-11-10T19:16:40,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a2d7c5bd484d4c6bbd47021d65d13ed9 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/a2d7c5bd484d4c6bbd47021d65d13ed9 2024-11-10T19:16:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T19:16:40,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/15599f8d9ab44c9d8680d839cbff8502 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/15599f8d9ab44c9d8680d839cbff8502 2024-11-10T19:16:40,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/10cfd09c5e5048d798ac6688422c7c98 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/10cfd09c5e5048d798ac6688422c7c98 2024-11-10T19:16:40,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/20a8dd3c23bf4dcfa6955e3e7f1a218e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/20a8dd3c23bf4dcfa6955e3e7f1a218e 2024-11-10T19:16:40,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d2d5e602062b4bf8a2517d35dccd8200 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/d2d5e602062b4bf8a2517d35dccd8200 2024-11-10T19:16:40,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e459877a06fc4931ac46523d7843a91b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e459877a06fc4931ac46523d7843a91b 2024-11-10T19:16:41,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43a6320633b9488181a44979dca5bfae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/77386700e04a4c00bd34eec8aef0924c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/dddb011cb6f24bbb8cb6944c0608bdf4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/bb81878f2e8146e282e18493455a13c4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1f1c95df63184aeeba84e56454870eed, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/46c72a6b2c3f49f7bc7c448fb2013de6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2edbdbe84b3d4f309dedfa1a446602cc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/88d6e80b880e46daae9a5fc7a41e0822, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/abfa7a3fcfda4e70aead0b967f9f01a3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6dbd68dcaf184e98b8101af3a54e1bcb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/fd68de2b5a044285a1e2772a7a4dead2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/66b6646fd8764dde8fe009909589c3c8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/b602d7b3a62e4a099c456e1ceacebdcb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/992a3e901d5644b0a5d26774d040845d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb47d989051461fb633e7c62f9d8e6e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1236070c1b5b4048990c849904812eb5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6ef81d2747284a96bcd82a240a9657a4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06c75f666e97446e818b92e4bf3070ce, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/5732939e2acf4e6597b6ff889ffd0a47, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/091618cfb0dc43f087dee50253839ad4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3f0ed730506f4658a72b9d6a1d5ead24, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06dc92b5144d4099a422ae510819dc0c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/35b760ddc09e45e182135a11c4daf166, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43ac8c96ed4a412681dea73d0fc2d6df, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/8fd6395497d149d2af9f2b28f7ceb816, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/de5164a4639e49afbf1f9128cc96a318, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3292101355b943ce9aaa3530ee10afff, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/aad0f5cd74cb445298da494747d2a1e2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d30aef4966b84fdfbc01c2f2b1e2ba1b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/e870382b858947ec80cee78a80caf224, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/c9491f20d2c549dfaff9d509adfe4361, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb40c8f7423427889bbd17f18cf7350, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/783096819712435da33dcaf1f2a72fc3] to archive 2024-11-10T19:16:41,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:16:41,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43a6320633b9488181a44979dca5bfae to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43a6320633b9488181a44979dca5bfae 2024-11-10T19:16:41,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/77386700e04a4c00bd34eec8aef0924c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/77386700e04a4c00bd34eec8aef0924c 2024-11-10T19:16:41,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/dddb011cb6f24bbb8cb6944c0608bdf4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/dddb011cb6f24bbb8cb6944c0608bdf4 2024-11-10T19:16:41,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/bb81878f2e8146e282e18493455a13c4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/bb81878f2e8146e282e18493455a13c4 2024-11-10T19:16:41,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1f1c95df63184aeeba84e56454870eed to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1f1c95df63184aeeba84e56454870eed 2024-11-10T19:16:41,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/46c72a6b2c3f49f7bc7c448fb2013de6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/46c72a6b2c3f49f7bc7c448fb2013de6 2024-11-10T19:16:41,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2edbdbe84b3d4f309dedfa1a446602cc to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2edbdbe84b3d4f309dedfa1a446602cc 2024-11-10T19:16:41,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/88d6e80b880e46daae9a5fc7a41e0822 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/88d6e80b880e46daae9a5fc7a41e0822 2024-11-10T19:16:41,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/abfa7a3fcfda4e70aead0b967f9f01a3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/abfa7a3fcfda4e70aead0b967f9f01a3 2024-11-10T19:16:41,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6dbd68dcaf184e98b8101af3a54e1bcb to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6dbd68dcaf184e98b8101af3a54e1bcb 2024-11-10T19:16:41,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/fd68de2b5a044285a1e2772a7a4dead2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/fd68de2b5a044285a1e2772a7a4dead2 2024-11-10T19:16:41,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/66b6646fd8764dde8fe009909589c3c8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/66b6646fd8764dde8fe009909589c3c8 2024-11-10T19:16:41,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/b602d7b3a62e4a099c456e1ceacebdcb to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/b602d7b3a62e4a099c456e1ceacebdcb 2024-11-10T19:16:41,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/992a3e901d5644b0a5d26774d040845d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/992a3e901d5644b0a5d26774d040845d 2024-11-10T19:16:41,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb47d989051461fb633e7c62f9d8e6e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb47d989051461fb633e7c62f9d8e6e 2024-11-10T19:16:41,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1236070c1b5b4048990c849904812eb5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/1236070c1b5b4048990c849904812eb5 2024-11-10T19:16:41,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6ef81d2747284a96bcd82a240a9657a4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/6ef81d2747284a96bcd82a240a9657a4 2024-11-10T19:16:41,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06c75f666e97446e818b92e4bf3070ce to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06c75f666e97446e818b92e4bf3070ce 2024-11-10T19:16:41,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/5732939e2acf4e6597b6ff889ffd0a47 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/5732939e2acf4e6597b6ff889ffd0a47 2024-11-10T19:16:41,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/091618cfb0dc43f087dee50253839ad4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/091618cfb0dc43f087dee50253839ad4 2024-11-10T19:16:41,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3f0ed730506f4658a72b9d6a1d5ead24 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3f0ed730506f4658a72b9d6a1d5ead24 2024-11-10T19:16:41,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06dc92b5144d4099a422ae510819dc0c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/06dc92b5144d4099a422ae510819dc0c 2024-11-10T19:16:41,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/35b760ddc09e45e182135a11c4daf166 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/35b760ddc09e45e182135a11c4daf166 2024-11-10T19:16:41,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43ac8c96ed4a412681dea73d0fc2d6df to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/43ac8c96ed4a412681dea73d0fc2d6df 2024-11-10T19:16:41,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/8fd6395497d149d2af9f2b28f7ceb816 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/8fd6395497d149d2af9f2b28f7ceb816 2024-11-10T19:16:41,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/de5164a4639e49afbf1f9128cc96a318 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/de5164a4639e49afbf1f9128cc96a318 2024-11-10T19:16:41,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3292101355b943ce9aaa3530ee10afff to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/3292101355b943ce9aaa3530ee10afff 2024-11-10T19:16:41,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/aad0f5cd74cb445298da494747d2a1e2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/aad0f5cd74cb445298da494747d2a1e2 2024-11-10T19:16:41,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d30aef4966b84fdfbc01c2f2b1e2ba1b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d30aef4966b84fdfbc01c2f2b1e2ba1b 2024-11-10T19:16:41,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/e870382b858947ec80cee78a80caf224 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/e870382b858947ec80cee78a80caf224 2024-11-10T19:16:41,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/c9491f20d2c549dfaff9d509adfe4361 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/c9491f20d2c549dfaff9d509adfe4361 2024-11-10T19:16:41,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb40c8f7423427889bbd17f18cf7350 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/2eb40c8f7423427889bbd17f18cf7350 2024-11-10T19:16:41,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/783096819712435da33dcaf1f2a72fc3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/783096819712435da33dcaf1f2a72fc3 2024-11-10T19:16:41,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/f9ae93e29fa44b1f909797b66121b124, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/40f570aa005a4ba595c8e22dbbaddb48, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e23dc7ddc9ac44739d7f240de5dd35bd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/1c9f3c1963e84fdfab2d6c2e57deaf9d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/487c373511ac43c4978a74af134571c5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3981ab2c416c4fafba049003e65409b8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/16c1dec03cdf474c8fd40f1cf8f21b10, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/46f15fdf2e964530a3649533de605032, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/c639da8620204d51836565cab5129b58, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d1d44b2a952d4c12877bee5145dab0f7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4dec4dd39e884375a48bd58acc0f0c16, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/85984b7e0c904a7e982f7e2c0898b7ba, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/82a00f727db749c580170f84db3e4a4d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/2c7b963df45c4026848a6b8af169dcc1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/83c318f239b14838a17cf8f38d97a3fc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5201823e57d643fa9983fde84241a425, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4c330065c1934320825bbd5033a88c89, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/69196693a2144258898ec07e58038b55, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/90d4cf0876084b6d96bb1185a505798c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9ba5b43d22ef4c78b4f40aa3e6901f50, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/92e4defbd18a4d5a971366d3e6a9b30f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/674d18e4edb14b80af3ae7b7797f1785, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9bf75ea8dd2c44e4b48759a073e402e1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dd69442c4b694be58c27263e177e2b2a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d020d502a5ac49ea979553274414987e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e6ec3012070240c6bcce682ef7ce6184, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/27ff56686513418abd48450d14fa597c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e3a4b15e0da64b2f998c6e5073ebf4be, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dc2ebe7953d4461ba3065b8e81a8099a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5f18c29a5a1f4a57aa95e1368c094503, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/bfce815c92d14bf1810129e99a65593f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/ed23da271e3b422aae246d7563444cd0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/09784408cefa4e2ca654a200c648d9de] to archive 2024-11-10T19:16:41,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:16:41,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/f9ae93e29fa44b1f909797b66121b124 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/f9ae93e29fa44b1f909797b66121b124 2024-11-10T19:16:41,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/40f570aa005a4ba595c8e22dbbaddb48 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/40f570aa005a4ba595c8e22dbbaddb48 2024-11-10T19:16:41,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e23dc7ddc9ac44739d7f240de5dd35bd to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e23dc7ddc9ac44739d7f240de5dd35bd 2024-11-10T19:16:41,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/1c9f3c1963e84fdfab2d6c2e57deaf9d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/1c9f3c1963e84fdfab2d6c2e57deaf9d 2024-11-10T19:16:41,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/487c373511ac43c4978a74af134571c5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/487c373511ac43c4978a74af134571c5 2024-11-10T19:16:41,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3981ab2c416c4fafba049003e65409b8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3981ab2c416c4fafba049003e65409b8 2024-11-10T19:16:41,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/16c1dec03cdf474c8fd40f1cf8f21b10 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/16c1dec03cdf474c8fd40f1cf8f21b10 2024-11-10T19:16:41,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/46f15fdf2e964530a3649533de605032 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/46f15fdf2e964530a3649533de605032 2024-11-10T19:16:41,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/c639da8620204d51836565cab5129b58 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/c639da8620204d51836565cab5129b58 2024-11-10T19:16:41,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d1d44b2a952d4c12877bee5145dab0f7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d1d44b2a952d4c12877bee5145dab0f7 2024-11-10T19:16:41,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4dec4dd39e884375a48bd58acc0f0c16 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4dec4dd39e884375a48bd58acc0f0c16 2024-11-10T19:16:41,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/85984b7e0c904a7e982f7e2c0898b7ba to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/85984b7e0c904a7e982f7e2c0898b7ba 2024-11-10T19:16:41,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/82a00f727db749c580170f84db3e4a4d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/82a00f727db749c580170f84db3e4a4d 2024-11-10T19:16:41,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/2c7b963df45c4026848a6b8af169dcc1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/2c7b963df45c4026848a6b8af169dcc1 2024-11-10T19:16:41,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/83c318f239b14838a17cf8f38d97a3fc to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/83c318f239b14838a17cf8f38d97a3fc 2024-11-10T19:16:41,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5201823e57d643fa9983fde84241a425 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5201823e57d643fa9983fde84241a425 2024-11-10T19:16:41,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4c330065c1934320825bbd5033a88c89 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/4c330065c1934320825bbd5033a88c89 2024-11-10T19:16:41,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/69196693a2144258898ec07e58038b55 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/69196693a2144258898ec07e58038b55 2024-11-10T19:16:41,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/90d4cf0876084b6d96bb1185a505798c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/90d4cf0876084b6d96bb1185a505798c 2024-11-10T19:16:41,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9ba5b43d22ef4c78b4f40aa3e6901f50 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9ba5b43d22ef4c78b4f40aa3e6901f50 2024-11-10T19:16:41,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/92e4defbd18a4d5a971366d3e6a9b30f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/92e4defbd18a4d5a971366d3e6a9b30f 2024-11-10T19:16:41,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/674d18e4edb14b80af3ae7b7797f1785 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/674d18e4edb14b80af3ae7b7797f1785 2024-11-10T19:16:41,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9bf75ea8dd2c44e4b48759a073e402e1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/9bf75ea8dd2c44e4b48759a073e402e1 2024-11-10T19:16:41,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dd69442c4b694be58c27263e177e2b2a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dd69442c4b694be58c27263e177e2b2a 2024-11-10T19:16:41,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d020d502a5ac49ea979553274414987e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/d020d502a5ac49ea979553274414987e 2024-11-10T19:16:41,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e6ec3012070240c6bcce682ef7ce6184 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e6ec3012070240c6bcce682ef7ce6184 2024-11-10T19:16:41,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/27ff56686513418abd48450d14fa597c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/27ff56686513418abd48450d14fa597c 2024-11-10T19:16:41,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e3a4b15e0da64b2f998c6e5073ebf4be to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/e3a4b15e0da64b2f998c6e5073ebf4be 2024-11-10T19:16:41,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dc2ebe7953d4461ba3065b8e81a8099a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/dc2ebe7953d4461ba3065b8e81a8099a 2024-11-10T19:16:41,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5f18c29a5a1f4a57aa95e1368c094503 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/5f18c29a5a1f4a57aa95e1368c094503 2024-11-10T19:16:41,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/bfce815c92d14bf1810129e99a65593f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/bfce815c92d14bf1810129e99a65593f 2024-11-10T19:16:41,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/ed23da271e3b422aae246d7563444cd0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/ed23da271e3b422aae246d7563444cd0 2024-11-10T19:16:41,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/09784408cefa4e2ca654a200c648d9de to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/09784408cefa4e2ca654a200c648d9de 2024-11-10T19:16:41,276 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/recovered.edits/459.seqid, newMaxSeqId=459, maxSeqId=1 2024-11-10T19:16:41,278 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. 2024-11-10T19:16:41,278 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] regionserver.HRegion(1676): Region close journal for 4f5e56bdcde85bbc88e3b9e5355a12b5: Waiting for close lock at 1731266200019Running coprocessor pre-close hooks at 1731266200019Disabling compacts and flushes for region at 1731266200019Disabling writes for close at 1731266200020 (+1 ms)Obtaining lock to block concurrent updates at 1731266200020Preparing flush snapshotting stores in 4f5e56bdcde85bbc88e3b9e5355a12b5 at 1731266200020Finished memstore snapshotting TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731266200021 (+1 ms)Flushing stores of TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5. at 1731266200022 (+1 ms)Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5/A: creating writer at 1731266200022Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5/A: appending metadata at 1731266200025 (+3 ms)Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5/A: closing flushed file at 1731266200025Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5/B: creating writer at 1731266200441 (+416 ms)Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5/B: appending metadata at 1731266200445 (+4 ms)Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5/B: closing flushed file at 1731266200445Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5/C: creating writer at 1731266200857 (+412 ms)Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5/C: appending metadata at 1731266200860 (+3 ms)Flushing 4f5e56bdcde85bbc88e3b9e5355a12b5/C: closing flushed file at 1731266200860Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6957c066: reopening flushed file at 1731266200872 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fee0542: reopening flushed file at 1731266200880 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10fcab91: reopening flushed file at 1731266200887 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4f5e56bdcde85bbc88e3b9e5355a12b5 in 876ms, sequenceid=456, compaction requested=true at 1731266200896 (+9 ms)Writing region close event to WAL at 1731266201232 (+336 ms)Running coprocessor post-close hooks at 1731266201278 (+46 ms)Closed at 1731266201278 2024-11-10T19:16:41,288 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=20}] handler.UnassignRegionHandler(157): Closed 4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:41,304 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=4f5e56bdcde85bbc88e3b9e5355a12b5, regionState=CLOSED 2024-11-10T19:16:41,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:16:41,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T19:16:41,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseRegionProcedure 4f5e56bdcde85bbc88e3b9e5355a12b5, server=bdc25462c9ef,34487,1731266167729 in 1.4720 sec 2024-11-10T19:16:41,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-10T19:16:41,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f5e56bdcde85bbc88e3b9e5355a12b5, UNASSIGN in 1.4830 sec 2024-11-10T19:16:41,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T19:16:41,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4960 sec 2024-11-10T19:16:41,351 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266201351"}]},"ts":"1731266201351"} 2024-11-10T19:16:41,357 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T19:16:41,357 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T19:16:41,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5200 sec 2024-11-10T19:16:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T19:16:42,002 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:42,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T19:16:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:42,014 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T19:16:42,018 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=21, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:42,026 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:42,033 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/recovered.edits] 2024-11-10T19:16:42,045 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/03faca5477094f5bbe8bb92c0df54bdd to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/03faca5477094f5bbe8bb92c0df54bdd 2024-11-10T19:16:42,047 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/50eaa165c306440d857872bca75b4ac3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/50eaa165c306440d857872bca75b4ac3 2024-11-10T19:16:42,049 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e20c69f5e0c146c883947f53c594187f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/A/e20c69f5e0c146c883947f53c594187f 2024-11-10T19:16:42,053 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/993f15e3883a4529b5f0cc1f55f705b6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/993f15e3883a4529b5f0cc1f55f705b6 2024-11-10T19:16:42,055 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/af3ef249ac9140a79d5c2d1c6bbbd0bf to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/af3ef249ac9140a79d5c2d1c6bbbd0bf 2024-11-10T19:16:42,062 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d84bc42bbe224a82a9f8d16a329b2310 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/B/d84bc42bbe224a82a9f8d16a329b2310 2024-11-10T19:16:42,067 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3118803e6ae249d193bee5ddecf8697d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3118803e6ae249d193bee5ddecf8697d 2024-11-10T19:16:42,070 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3947e30e52964116ab111eb1d1fb7158 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/3947e30e52964116ab111eb1d1fb7158 2024-11-10T19:16:42,073 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/8d4cc20210bb4289a8300a1464abdc15 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/C/8d4cc20210bb4289a8300a1464abdc15 2024-11-10T19:16:42,079 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/recovered.edits/459.seqid to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5/recovered.edits/459.seqid 2024-11-10T19:16:42,080 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4f5e56bdcde85bbc88e3b9e5355a12b5 2024-11-10T19:16:42,080 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T19:16:42,087 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=21, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:42,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T19:16:42,102 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T19:16:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T19:16:42,156 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T19:16:42,158 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=21, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:42,158 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T19:16:42,158 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731266202158"}]},"ts":"9223372036854775807"} 2024-11-10T19:16:42,165 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T19:16:42,165 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4f5e56bdcde85bbc88e3b9e5355a12b5, NAME => 'TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T19:16:42,166 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T19:16:42,166 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731266202166"}]},"ts":"9223372036854775807"} 2024-11-10T19:16:42,178 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T19:16:42,180 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=21, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:42,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 173 msec 2024-11-10T19:16:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T19:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T19:16:42,333 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:42,345 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=240 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;bdc25462c9ef:34487-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1070050616_22 at /127.0.0.1:45822 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=722 (was 593) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2664 (was 3499) 2024-11-10T19:16:42,362 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=722, ProcessCount=11, AvailableMemoryMB=2663 2024-11-10T19:16:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T19:16:42,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:16:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:42,380 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T19:16:42,380 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:42,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 22 2024-11-10T19:16:42,383 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T19:16:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-10T19:16:42,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741945_1121 (size=1061) 2024-11-10T19:16:42,446 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0 2024-11-10T19:16:42,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741946_1122 (size=53) 2024-11-10T19:16:42,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:16:42,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 17e63e9bb9f2380be13ccc4351250608, disabling compactions & flushes 2024-11-10T19:16:42,485 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:42,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:42,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. after waiting 0 ms 2024-11-10T19:16:42,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:42,485 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:42,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17e63e9bb9f2380be13ccc4351250608: Waiting for close lock at 1731266202485Disabling compacts and flushes for region at 1731266202485Disabling writes for close at 1731266202485Writing region close event to WAL at 1731266202485Closed at 1731266202485 2024-11-10T19:16:42,488 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T19:16:42,488 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731266202488"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731266202488"}]},"ts":"1731266202488"} 2024-11-10T19:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-10T19:16:42,492 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T19:16:42,494 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T19:16:42,494 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266202494"}]},"ts":"1731266202494"} 2024-11-10T19:16:42,498 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T19:16:42,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, ASSIGN}] 2024-11-10T19:16:42,500 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, ASSIGN 2024-11-10T19:16:42,502 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, ASSIGN; state=OFFLINE, location=bdc25462c9ef,34487,1731266167729; forceNewPlan=false, retain=false 2024-11-10T19:16:42,654 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=17e63e9bb9f2380be13ccc4351250608, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:42,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, ASSIGN because future has completed 2024-11-10T19:16:42,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:16:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-10T19:16:42,822 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:42,822 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:16:42,822 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,823 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:16:42,823 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,823 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,824 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,825 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:16:42,826 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17e63e9bb9f2380be13ccc4351250608 columnFamilyName A 2024-11-10T19:16:42,827 DEBUG [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:42,829 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(327): Store=17e63e9bb9f2380be13ccc4351250608/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:42,829 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,836 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:16:42,837 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17e63e9bb9f2380be13ccc4351250608 columnFamilyName B 2024-11-10T19:16:42,837 DEBUG [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:42,840 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(327): Store=17e63e9bb9f2380be13ccc4351250608/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:42,841 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,842 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:16:42,842 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17e63e9bb9f2380be13ccc4351250608 columnFamilyName C 2024-11-10T19:16:42,842 DEBUG [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:42,843 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(327): Store=17e63e9bb9f2380be13ccc4351250608/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:42,843 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:42,843 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,844 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,844 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,845 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,845 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,846 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:16:42,847 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,850 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:16:42,850 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 17e63e9bb9f2380be13ccc4351250608; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75372422, jitterRate=0.12313660979270935}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:16:42,850 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:42,851 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 17e63e9bb9f2380be13ccc4351250608: Running coprocessor pre-open hook at 1731266202823Writing region info on filesystem at 1731266202823Initializing all the Stores at 1731266202824 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266202824Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266202824Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266202824Cleaning up temporary data from old regions at 1731266202845 (+21 ms)Running coprocessor post-open hooks at 1731266202850 (+5 ms)Region opened successfully at 1731266202851 (+1 ms) 2024-11-10T19:16:42,853 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., pid=24, masterSystemTime=1731266202818 2024-11-10T19:16:42,856 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:42,856 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:42,857 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=17e63e9bb9f2380be13ccc4351250608, regionState=OPEN, openSeqNum=2, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:42,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:16:42,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T19:16:42,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 in 199 msec 2024-11-10T19:16:42,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-10T19:16:42,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, ASSIGN in 367 msec 2024-11-10T19:16:42,870 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T19:16:42,870 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266202870"}]},"ts":"1731266202870"} 2024-11-10T19:16:42,875 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T19:16:42,877 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T19:16:42,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 506 msec 2024-11-10T19:16:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-10T19:16:43,012 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:43,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:43,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:43,015 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c45a5be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:43,018 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb71261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,019 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:43,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:43,023 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T19:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is bdc25462c9ef,46761,1731266166843 2024-11-10T19:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45945b10 2024-11-10T19:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T19:16:43,025 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T19:16:43,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T19:16:43,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:16:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T19:16:43,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741947_1123 (size=1097) 2024-11-10T19:16:43,448 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T19:16:43,449 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T19:16:43,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T19:16:43,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, REOPEN/MOVE}] 2024-11-10T19:16:43,460 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, REOPEN/MOVE 2024-11-10T19:16:43,460 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=17e63e9bb9f2380be13ccc4351250608, regionState=CLOSING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:43,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, REOPEN/MOVE because future has completed 2024-11-10T19:16:43,463 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T19:16:43,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:16:43,616 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,616 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T19:16:43,616 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 17e63e9bb9f2380be13ccc4351250608, disabling compactions & flushes 2024-11-10T19:16:43,616 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:43,616 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:43,616 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. after waiting 0 ms 2024-11-10T19:16:43,616 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:43,621 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T19:16:43,622 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:43,622 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 17e63e9bb9f2380be13ccc4351250608: Waiting for close lock at 1731266203616Running coprocessor pre-close hooks at 1731266203616Disabling compacts and flushes for region at 1731266203616Disabling writes for close at 1731266203616Writing region close event to WAL at 1731266203617 (+1 ms)Running coprocessor post-close hooks at 1731266203622 (+5 ms)Closed at 1731266203622 2024-11-10T19:16:43,622 WARN [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionServer(3299): Not adding moved region record: 17e63e9bb9f2380be13ccc4351250608 to self. 2024-11-10T19:16:43,624 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,625 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=17e63e9bb9f2380be13ccc4351250608, regionState=CLOSED 2024-11-10T19:16:43,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:16:43,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-10T19:16:43,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 in 165 msec 2024-11-10T19:16:43,631 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, REOPEN/MOVE; state=CLOSED, location=bdc25462c9ef,34487,1731266167729; forceNewPlan=false, retain=true 2024-11-10T19:16:43,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=17e63e9bb9f2380be13ccc4351250608, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:43,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, REOPEN/MOVE because future has completed 2024-11-10T19:16:43,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:16:43,947 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:43,947 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7752): Opening region: {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:16:43,948 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,948 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:16:43,948 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7794): checking encryption for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,948 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7797): checking classloading for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,951 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,951 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:16:43,957 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17e63e9bb9f2380be13ccc4351250608 columnFamilyName A 2024-11-10T19:16:43,959 DEBUG [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:43,959 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(327): Store=17e63e9bb9f2380be13ccc4351250608/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:43,960 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,960 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:16:43,960 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17e63e9bb9f2380be13ccc4351250608 columnFamilyName B 2024-11-10T19:16:43,960 DEBUG [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:43,961 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(327): Store=17e63e9bb9f2380be13ccc4351250608/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:43,961 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,961 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:16:43,961 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17e63e9bb9f2380be13ccc4351250608 columnFamilyName C 2024-11-10T19:16:43,962 DEBUG [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:43,962 INFO [StoreOpener-17e63e9bb9f2380be13ccc4351250608-1 {}] regionserver.HStore(327): Store=17e63e9bb9f2380be13ccc4351250608/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:16:43,962 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:43,962 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1038): replaying wal for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,963 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,964 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,964 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1048): stopping wal replay for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,964 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1060): Cleaning up temporary data for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,965 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:16:43,966 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): writing seq id for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,966 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1114): Opened 17e63e9bb9f2380be13ccc4351250608; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63924744, jitterRate=-0.0474470853805542}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:16:43,967 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:43,968 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1006): Region open journal for 17e63e9bb9f2380be13ccc4351250608: Running coprocessor pre-open hook at 1731266203948Writing region info on filesystem at 1731266203948Initializing all the Stores at 1731266203949 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266203949Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266203949Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266203949Cleaning up temporary data from old regions at 1731266203964 (+15 ms)Running coprocessor post-open hooks at 1731266203967 (+3 ms)Region opened successfully at 1731266203968 (+1 ms) 2024-11-10T19:16:43,969 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., pid=29, masterSystemTime=1731266203942 2024-11-10T19:16:43,971 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:43,971 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:43,971 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=17e63e9bb9f2380be13ccc4351250608, regionState=OPEN, openSeqNum=5, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:43,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:16:43,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=27 2024-11-10T19:16:43,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 in 187 msec 2024-11-10T19:16:43,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-10T19:16:43,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, REOPEN/MOVE in 516 msec 2024-11-10T19:16:43,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-10T19:16:43,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-11-10T19:16:43,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 949 msec 2024-11-10T19:16:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T19:16:43,985 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:43,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:43,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:43,986 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16abf9a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:43,988 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d89034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:43,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:43,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:43,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5794bfb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:43,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:43,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71132f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:43,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52148bba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:43,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:43,999 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@274caf2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:44,001 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38909483, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:44,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:44,003 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad1f224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:44,005 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a67db5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:44,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:44,007 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2446c366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:44,009 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac12f0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:44,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:44,012 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa6b1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:44,013 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f2083a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:44,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:44,016 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c495221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:44,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:44,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2d9821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:16:44,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:16:44,019 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:16:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37e6f53e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:16:44,021 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:16:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:16:44,022 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:44,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees 2024-11-10T19:16:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T19:16:44,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:44,025 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=30, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:44,025 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=30, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:44,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:16:44,029 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:44,032 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:44,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:44,033 DEBUG [Thread-639 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:44,033 DEBUG [Thread-639 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:44,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:16:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:16:44,047 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:44,051 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:44,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:16:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:44,061 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:44,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:44,062 DEBUG [Thread-637 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:44,063 DEBUG [Thread-637 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:44,064 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:44,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:16:44,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:16:44,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:44,065 DEBUG [Thread-630 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:44,065 DEBUG [Thread-630 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:44,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:16:44,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:16:44,080 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:44,082 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:16:44,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:16:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:16:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:16:44,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:16:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:16:44,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111009c5eb2be5944d43b165df37ac289a8a_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266204055/Put/seqid=0 2024-11-10T19:16:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T19:16:44,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731266204094, max timestamp=1731266204127Num uniques 859; 2024-11-10T19:16:44,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T19:16:44,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731266204094, max timestamp=1731266204127Num uniques 859; 2024-11-10T19:16:44,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T19:16:44,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731266204094, max timestamp=1731266204128Num uniques 859; 2024-11-10T19:16:44,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T19:16:44,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-10T19:16:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:44,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(58): pid=31 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:44,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741948_1124 (size=12154) 2024-11-10T19:16:44,189 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:44,199 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111009c5eb2be5944d43b165df37ac289a8a_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111009c5eb2be5944d43b165df37ac289a8a_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:44,201 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/bf4c65acd6b741ac8d3f9513507f94b8, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:44,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266264191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/bf4c65acd6b741ac8d3f9513507f94b8 is 175, key is test_row_0/A:col10/1731266204055/Put/seqid=0 2024-11-10T19:16:44,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266264203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266264208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266264211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266264212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741949_1125 (size=30955) 2024-11-10T19:16:44,235 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/bf4c65acd6b741ac8d3f9513507f94b8 2024-11-10T19:16:44,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/c2c17b1332904883b5b04bb4224a7f1d is 50, key is test_row_0/B:col10/1731266204055/Put/seqid=0 2024-11-10T19:16:44,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741950_1126 (size=12001) 2024-11-10T19:16:44,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/c2c17b1332904883b5b04bb4224a7f1d 2024-11-10T19:16:44,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/5df3ac73de9f408f9181903e5de20f32 is 50, key is test_row_0/C:col10/1731266204055/Put/seqid=0 2024-11-10T19:16:44,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266264321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266264321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-10T19:16:44,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:44,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:44,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:44,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(58): pid=31 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:44,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T19:16:44,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741951_1127 (size=12001) 2024-11-10T19:16:44,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266264332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266264332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266264332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/5df3ac73de9f408f9181903e5de20f32 2024-11-10T19:16:44,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/bf4c65acd6b741ac8d3f9513507f94b8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/bf4c65acd6b741ac8d3f9513507f94b8 2024-11-10T19:16:44,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/bf4c65acd6b741ac8d3f9513507f94b8, entries=150, sequenceid=14, filesize=30.2 K 2024-11-10T19:16:44,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/c2c17b1332904883b5b04bb4224a7f1d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c2c17b1332904883b5b04bb4224a7f1d 2024-11-10T19:16:44,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c2c17b1332904883b5b04bb4224a7f1d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T19:16:44,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/5df3ac73de9f408f9181903e5de20f32 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5df3ac73de9f408f9181903e5de20f32 2024-11-10T19:16:44,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5df3ac73de9f408f9181903e5de20f32, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T19:16:44,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 17e63e9bb9f2380be13ccc4351250608 in 337ms, sequenceid=14, compaction requested=false 2024-11-10T19:16:44,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:44,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-10T19:16:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:44,493 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-10T19:16:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:16:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:44,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:16:44,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:44,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:16:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106ed24c01568f415bbb1529b00d4d56c8_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266204176/Put/seqid=0 2024-11-10T19:16:44,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741952_1128 (size=12154) 2024-11-10T19:16:44,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:44,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:44,567 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106ed24c01568f415bbb1529b00d4d56c8_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106ed24c01568f415bbb1529b00d4d56c8_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:44,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:44,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b is 175, key is test_row_0/A:col10/1731266204176/Put/seqid=0 2024-11-10T19:16:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266264562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266264574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266264580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741953_1129 (size=30955) 2024-11-10T19:16:44,612 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b 2024-11-10T19:16:44,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266264593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266264608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/86c78ff610904b758bd991ba0ea84d55 is 50, key is test_row_0/B:col10/1731266204176/Put/seqid=0 2024-11-10T19:16:44,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741954_1130 (size=12001) 2024-11-10T19:16:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T19:16:44,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266264711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266264721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266264721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266264732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266264901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266264922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266264956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266264956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:44,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266264956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:44,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:44,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,052 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/86c78ff610904b758bd991ba0ea84d55 2024-11-10T19:16:45,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/11e0f26ab7484bb594ec96bf709db49f is 50, key is test_row_0/C:col10/1731266204176/Put/seqid=0 2024-11-10T19:16:45,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741955_1131 (size=12001) 2024-11-10T19:16:45,098 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/11e0f26ab7484bb594ec96bf709db49f 2024-11-10T19:16:45,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b 2024-11-10T19:16:45,117 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b, entries=150, sequenceid=50, filesize=30.2 K 2024-11-10T19:16:45,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/86c78ff610904b758bd991ba0ea84d55 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/86c78ff610904b758bd991ba0ea84d55 2024-11-10T19:16:45,139 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/86c78ff610904b758bd991ba0ea84d55, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T19:16:45,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/11e0f26ab7484bb594ec96bf709db49f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/11e0f26ab7484bb594ec96bf709db49f 2024-11-10T19:16:45,151 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/11e0f26ab7484bb594ec96bf709db49f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T19:16:45,152 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 17e63e9bb9f2380be13ccc4351250608 in 659ms, sequenceid=50, compaction requested=false 2024-11-10T19:16:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=31 2024-11-10T19:16:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=31 2024-11-10T19:16:45,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-10T19:16:45,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1290 sec 2024-11-10T19:16:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T19:16:45,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees in 1.1380 sec 2024-11-10T19:16:45,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:16:45,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:45,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:45,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:45,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:45,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:45,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:45,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110997222cb53bb47cea73355e96f70057e_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266204584/Put/seqid=0 2024-11-10T19:16:45,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741956_1132 (size=14594) 2024-11-10T19:16:45,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266205317, max timestamp=1731266205331Num uniques 859; 2024-11-10T19:16:45,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:16:45,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266205317, max timestamp=1731266205331Num uniques 859; 2024-11-10T19:16:45,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:16:45,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266205317, max timestamp=1731266205331Num uniques 859; 2024-11-10T19:16:45,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:16:45,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266265397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266265398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731266205376, max timestamp=1731266205405Num uniques 1018; 2024-11-10T19:16:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T19:16:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731266205376, max timestamp=1731266205405Num uniques 1018; 2024-11-10T19:16:45,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T19:16:45,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731266205376, max timestamp=1731266205405Num uniques 1018; 2024-11-10T19:16:45,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T19:16:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266265474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266265477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266265479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266265521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266265521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266265591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266265591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266265591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,707 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:45,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110997222cb53bb47cea73355e96f70057e_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110997222cb53bb47cea73355e96f70057e_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:45,718 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/ef439ec78a114bdeb19e3af194478309, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:45,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/ef439ec78a114bdeb19e3af194478309 is 175, key is test_row_0/A:col10/1731266204584/Put/seqid=0 2024-11-10T19:16:45,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266265732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266265732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741957_1133 (size=39549) 2024-11-10T19:16:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266265802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266265802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266265802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266266049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266266049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266266112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266266112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266266116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,148 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/ef439ec78a114bdeb19e3af194478309 2024-11-10T19:16:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T19:16:46,171 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:46,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-10T19:16:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T19:16:46,174 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:46,175 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:46,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/f1b9f2c46d2b4212927437f000798d88 is 50, key is test_row_0/B:col10/1731266204584/Put/seqid=0 2024-11-10T19:16:46,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:46,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741958_1134 (size=12001) 2024-11-10T19:16:46,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/f1b9f2c46d2b4212927437f000798d88 2024-11-10T19:16:46,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/46ea20317a7c463492a3885be38efc44 is 50, key is test_row_0/C:col10/1731266204584/Put/seqid=0 2024-11-10T19:16:46,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741959_1135 (size=12001) 2024-11-10T19:16:46,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/46ea20317a7c463492a3885be38efc44 2024-11-10T19:16:46,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/ef439ec78a114bdeb19e3af194478309 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/ef439ec78a114bdeb19e3af194478309 2024-11-10T19:16:46,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/ef439ec78a114bdeb19e3af194478309, entries=200, sequenceid=60, filesize=38.6 K 2024-11-10T19:16:46,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/f1b9f2c46d2b4212927437f000798d88 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f1b9f2c46d2b4212927437f000798d88 2024-11-10T19:16:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T19:16:46,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f1b9f2c46d2b4212927437f000798d88, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T19:16:46,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/46ea20317a7c463492a3885be38efc44 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/46ea20317a7c463492a3885be38efc44 2024-11-10T19:16:46,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/46ea20317a7c463492a3885be38efc44, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T19:16:46,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 17e63e9bb9f2380be13ccc4351250608 in 1067ms, sequenceid=60, compaction requested=true 2024-11-10T19:16:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:46,304 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:46,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:46,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T19:16:46,305 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:46,309 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:46,309 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:46,309 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:16:46,309 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:16:46,309 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:46,309 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c2c17b1332904883b5b04bb4224a7f1d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/86c78ff610904b758bd991ba0ea84d55, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f1b9f2c46d2b4212927437f000798d88] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=35.2 K 2024-11-10T19:16:46,310 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:46,310 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/bf4c65acd6b741ac8d3f9513507f94b8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/ef439ec78a114bdeb19e3af194478309] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=99.1 K 2024-11-10T19:16:46,310 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c17b1332904883b5b04bb4224a7f1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731266204055 2024-11-10T19:16:46,310 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:46,310 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/bf4c65acd6b741ac8d3f9513507f94b8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/ef439ec78a114bdeb19e3af194478309] 2024-11-10T19:16:46,311 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf4c65acd6b741ac8d3f9513507f94b8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731266204055 2024-11-10T19:16:46,311 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c78ff610904b758bd991ba0ea84d55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731266204176 2024-11-10T19:16:46,311 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting f1b9f2c46d2b4212927437f000798d88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731266204553 2024-11-10T19:16:46,312 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc7ecb3a13b4b01b1bb70c8b9861e4b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731266204176 2024-11-10T19:16:46,313 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef439ec78a114bdeb19e3af194478309, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731266204553 2024-11-10T19:16:46,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-10T19:16:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:46,331 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=134.18 KB heapSize=374.77 KB 2024-11-10T19:16:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:16:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:16:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:16:46,339 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:46,341 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/0e76671b2c384f0588d78cd13d99df71 is 50, key is test_row_0/B:col10/1731266204584/Put/seqid=0 2024-11-10T19:16:46,346 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:46,374 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111052ecd83216ab47f38512894d13fb4809_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:46,380 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111052ecd83216ab47f38512894d13fb4809_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:46,380 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111052ecd83216ab47f38512894d13fb4809_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:46,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102e1b9b35648d4d78bf4f4a22658932c2_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266205474/Put/seqid=0 2024-11-10T19:16:46,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741960_1136 (size=12104) 2024-11-10T19:16:46,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741961_1137 (size=4469) 2024-11-10T19:16:46,433 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#118 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:46,434 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/0eaf46fded8948cca2f2fd1051e74445 is 175, key is test_row_0/A:col10/1731266204584/Put/seqid=0 2024-11-10T19:16:46,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741962_1138 (size=12154) 2024-11-10T19:16:46,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:46,441 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102e1b9b35648d4d78bf4f4a22658932c2_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e1b9b35648d4d78bf4f4a22658932c2_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:46,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8bdc231c4fa04ad8a104bf47024810da, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8bdc231c4fa04ad8a104bf47024810da is 175, key is test_row_0/A:col10/1731266205474/Put/seqid=0 2024-11-10T19:16:46,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741963_1139 (size=31058) 2024-11-10T19:16:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T19:16:46,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741964_1140 (size=30955) 2024-11-10T19:16:46,506 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8bdc231c4fa04ad8a104bf47024810da 2024-11-10T19:16:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/457adaae175c4e048a687d8014fb7351 is 50, key is test_row_0/B:col10/1731266205474/Put/seqid=0 2024-11-10T19:16:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:46,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:46,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741965_1141 (size=12001) 2024-11-10T19:16:46,591 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/457adaae175c4e048a687d8014fb7351 2024-11-10T19:16:46,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266266594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266266600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/1780c1581eb049d9801067bcf1de9cb5 is 50, key is test_row_0/C:col10/1731266205474/Put/seqid=0 2024-11-10T19:16:46,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266266631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266266632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266266632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741966_1142 (size=12001) 2024-11-10T19:16:46,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266266712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266266712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T19:16:46,834 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/0e76671b2c384f0588d78cd13d99df71 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/0e76671b2c384f0588d78cd13d99df71 2024-11-10T19:16:46,845 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into 0e76671b2c384f0588d78cd13d99df71(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:46,845 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:46,845 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266206304; duration=0sec 2024-11-10T19:16:46,845 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:46,845 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:16:46,845 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:46,847 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:46,848 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:16:46,848 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:46,849 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5df3ac73de9f408f9181903e5de20f32, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/11e0f26ab7484bb594ec96bf709db49f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/46ea20317a7c463492a3885be38efc44] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=35.2 K 2024-11-10T19:16:46,850 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df3ac73de9f408f9181903e5de20f32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731266204055 2024-11-10T19:16:46,850 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e0f26ab7484bb594ec96bf709db49f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731266204176 2024-11-10T19:16:46,851 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 46ea20317a7c463492a3885be38efc44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731266204553 2024-11-10T19:16:46,869 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#122 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:46,871 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/0310198fc3b04570815a32efa2959326 is 50, key is test_row_0/C:col10/1731266204584/Put/seqid=0 2024-11-10T19:16:46,888 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/0eaf46fded8948cca2f2fd1051e74445 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0eaf46fded8948cca2f2fd1051e74445 2024-11-10T19:16:46,897 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into 0eaf46fded8948cca2f2fd1051e74445(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:46,897 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:46,897 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266206304; duration=0sec 2024-11-10T19:16:46,897 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:46,897 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:16:46,915 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T19:16:46,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741967_1143 (size=12104) 2024-11-10T19:16:46,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266266933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266266935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,052 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/1780c1581eb049d9801067bcf1de9cb5 2024-11-10T19:16:47,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8bdc231c4fa04ad8a104bf47024810da as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8bdc231c4fa04ad8a104bf47024810da 2024-11-10T19:16:47,064 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8bdc231c4fa04ad8a104bf47024810da, entries=150, sequenceid=112, filesize=30.2 K 2024-11-10T19:16:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/457adaae175c4e048a687d8014fb7351 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/457adaae175c4e048a687d8014fb7351 2024-11-10T19:16:47,070 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/457adaae175c4e048a687d8014fb7351, entries=150, sequenceid=112, filesize=11.7 K 2024-11-10T19:16:47,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/1780c1581eb049d9801067bcf1de9cb5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1780c1581eb049d9801067bcf1de9cb5 2024-11-10T19:16:47,102 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1780c1581eb049d9801067bcf1de9cb5, entries=150, sequenceid=112, filesize=11.7 K 2024-11-10T19:16:47,104 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~374.72 KB/383712, currentSize=46.96 KB/48090 for 17e63e9bb9f2380be13ccc4351250608 in 772ms, sequenceid=112, compaction requested=false 2024-11-10T19:16:47,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:47,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:47,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-10T19:16:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-10T19:16:47,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-10T19:16:47,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 932 msec 2024-11-10T19:16:47,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 939 msec 2024-11-10T19:16:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:47,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T19:16:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:47,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110473e506fc83140258fa84ffaaee5c373_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266207261/Put/seqid=0 2024-11-10T19:16:47,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741968_1144 (size=17034) 2024-11-10T19:16:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T19:16:47,312 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:47,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-10T19:16:47,317 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T19:16:47,318 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:47,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:47,335 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/0310198fc3b04570815a32efa2959326 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0310198fc3b04570815a32efa2959326 2024-11-10T19:16:47,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266267331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266267335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,346 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into 0310198fc3b04570815a32efa2959326(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:47,347 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:47,347 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266206304; duration=0sec 2024-11-10T19:16:47,348 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:47,348 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:16:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T19:16:47,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266267442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266267442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T19:16:47,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:47,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:47,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:47,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T19:16:47,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:47,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T19:16:47,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266267641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266267641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,644 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,644 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266267651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266267651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,654 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:47,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266267651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:47,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110473e506fc83140258fa84ffaaee5c373_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110473e506fc83140258fa84ffaaee5c373_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:47,703 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7a22be4965bf48b99429831cf4aa3e71, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:47,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7a22be4965bf48b99429831cf4aa3e71 is 175, key is test_row_0/A:col10/1731266207261/Put/seqid=0 2024-11-10T19:16:47,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741969_1145 (size=48139) 2024-11-10T19:16:47,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T19:16:47,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:47,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:47,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:47,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T19:16:47,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:47,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:47,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:47,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T19:16:47,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266267961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:47,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266267962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:48,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T19:16:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,111 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=126, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7a22be4965bf48b99429831cf4aa3e71 2024-11-10T19:16:48,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/33ecdcb2e21a42eda3385aecb670b83b is 50, key is test_row_0/B:col10/1731266207261/Put/seqid=0 2024-11-10T19:16:48,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741970_1146 (size=12001) 2024-11-10T19:16:48,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/33ecdcb2e21a42eda3385aecb670b83b 2024-11-10T19:16:48,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/dc7ae2f2840f482398b62429d2e1c984 is 50, key is test_row_0/C:col10/1731266207261/Put/seqid=0 2024-11-10T19:16:48,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741971_1147 (size=12001) 2024-11-10T19:16:48,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T19:16:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T19:16:48,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:48,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T19:16:48,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266268471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:48,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266268471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:48,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:48,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:48,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T19:16:48,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:48,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:48,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/dc7ae2f2840f482398b62429d2e1c984 2024-11-10T19:16:48,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7a22be4965bf48b99429831cf4aa3e71 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7a22be4965bf48b99429831cf4aa3e71 2024-11-10T19:16:48,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7a22be4965bf48b99429831cf4aa3e71, entries=250, sequenceid=126, filesize=47.0 K 2024-11-10T19:16:48,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/33ecdcb2e21a42eda3385aecb670b83b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/33ecdcb2e21a42eda3385aecb670b83b 2024-11-10T19:16:48,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/33ecdcb2e21a42eda3385aecb670b83b, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T19:16:48,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/dc7ae2f2840f482398b62429d2e1c984 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/dc7ae2f2840f482398b62429d2e1c984 2024-11-10T19:16:48,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/dc7ae2f2840f482398b62429d2e1c984, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T19:16:48,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 17e63e9bb9f2380be13ccc4351250608 in 1443ms, sequenceid=126, compaction requested=true 2024-11-10T19:16:48,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:48,707 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:48,708 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:48,709 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:48,709 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:16:48,709 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,709 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0eaf46fded8948cca2f2fd1051e74445, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8bdc231c4fa04ad8a104bf47024810da, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7a22be4965bf48b99429831cf4aa3e71] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=107.6 K 2024-11-10T19:16:48,710 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,710 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0eaf46fded8948cca2f2fd1051e74445, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8bdc231c4fa04ad8a104bf47024810da, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7a22be4965bf48b99429831cf4aa3e71] 2024-11-10T19:16:48,710 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eaf46fded8948cca2f2fd1051e74445, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731266204553 2024-11-10T19:16:48,710 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:48,711 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:16:48,711 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,711 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/0e76671b2c384f0588d78cd13d99df71, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/457adaae175c4e048a687d8014fb7351, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/33ecdcb2e21a42eda3385aecb670b83b] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=35.3 K 2024-11-10T19:16:48,711 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bdc231c4fa04ad8a104bf47024810da, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731266205440 2024-11-10T19:16:48,712 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e76671b2c384f0588d78cd13d99df71, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731266204553 2024-11-10T19:16:48,712 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a22be4965bf48b99429831cf4aa3e71, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731266206582 2024-11-10T19:16:48,713 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 457adaae175c4e048a687d8014fb7351, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731266205440 2024-11-10T19:16:48,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-10T19:16:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,715 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ecdcb2e21a42eda3385aecb670b83b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731266206592 2024-11-10T19:16:48,715 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T19:16:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:48,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103539502e62654e5ba00144eb0dd0f08e_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266207325/Put/seqid=0 2024-11-10T19:16:48,747 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#127 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:48,748 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/7fb5a54eb14245bd9309c3c5a067f340 is 50, key is test_row_0/B:col10/1731266207261/Put/seqid=0 2024-11-10T19:16:48,761 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:48,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741972_1148 (size=12304) 2024-11-10T19:16:48,800 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411108a050b92e04c4d43bc77b437cb218263_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:48,804 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411108a050b92e04c4d43bc77b437cb218263_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:48,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741973_1149 (size=12207) 2024-11-10T19:16:48,804 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108a050b92e04c4d43bc77b437cb218263_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:48,823 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/7fb5a54eb14245bd9309c3c5a067f340 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/7fb5a54eb14245bd9309c3c5a067f340 2024-11-10T19:16:48,847 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into 7fb5a54eb14245bd9309c3c5a067f340(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:48,848 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:48,848 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266208708; duration=0sec 2024-11-10T19:16:48,848 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:48,848 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:16:48,848 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:48,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741974_1150 (size=4469) 2024-11-10T19:16:48,853 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#128 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:48,854 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:48,854 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:16:48,854 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:48,854 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0310198fc3b04570815a32efa2959326, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1780c1581eb049d9801067bcf1de9cb5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/dc7ae2f2840f482398b62429d2e1c984] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=35.3 K 2024-11-10T19:16:48,855 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 0310198fc3b04570815a32efa2959326, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731266204553 2024-11-10T19:16:48,855 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/defd1d28a23140d3a24203f364326652 is 175, key is test_row_0/A:col10/1731266207261/Put/seqid=0 2024-11-10T19:16:48,856 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 1780c1581eb049d9801067bcf1de9cb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731266205440 2024-11-10T19:16:48,858 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting dc7ae2f2840f482398b62429d2e1c984, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731266206592 2024-11-10T19:16:48,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741975_1151 (size=31161) 2024-11-10T19:16:48,903 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:48,903 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/defd1d28a23140d3a24203f364326652 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/defd1d28a23140d3a24203f364326652 2024-11-10T19:16:48,903 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/33d04d31afe24c1f9a1cd47454a93004 is 50, key is test_row_0/C:col10/1731266207261/Put/seqid=0 2024-11-10T19:16:48,917 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into defd1d28a23140d3a24203f364326652(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:48,917 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:48,917 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266208707; duration=0sec 2024-11-10T19:16:48,917 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:48,918 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:16:48,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741976_1152 (size=12207) 2024-11-10T19:16:48,943 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/33d04d31afe24c1f9a1cd47454a93004 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/33d04d31afe24c1f9a1cd47454a93004 2024-11-10T19:16:48,957 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into 33d04d31afe24c1f9a1cd47454a93004(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:48,957 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:48,958 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266208708; duration=0sec 2024-11-10T19:16:48,959 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:48,959 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:16:49,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:49,184 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103539502e62654e5ba00144eb0dd0f08e_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103539502e62654e5ba00144eb0dd0f08e_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:49,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/6a159d35426d452498e19d36602da178, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:49,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/6a159d35426d452498e19d36602da178 is 175, key is test_row_0/A:col10/1731266207325/Put/seqid=0 2024-11-10T19:16:49,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741977_1153 (size=31105) 2024-11-10T19:16:49,215 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/6a159d35426d452498e19d36602da178 2024-11-10T19:16:49,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/178aee35174b4726bcd9cf8a6c4b5b6a is 50, key is test_row_0/B:col10/1731266207325/Put/seqid=0 2024-11-10T19:16:49,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741978_1154 (size=12151) 2024-11-10T19:16:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T19:16:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:49,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266269542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:49,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266269558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:49,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:49,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266269652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:49,663 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:49,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266269666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:49,669 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:49,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266269667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:49,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266269672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266269672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:49,685 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:49,699 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/178aee35174b4726bcd9cf8a6c4b5b6a 2024-11-10T19:16:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/fdd2098f8570482ca881ae90d23ecdf3 is 50, key is test_row_0/C:col10/1731266207325/Put/seqid=0 2024-11-10T19:16:49,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741979_1155 (size=12151) 2024-11-10T19:16:49,784 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/fdd2098f8570482ca881ae90d23ecdf3 2024-11-10T19:16:49,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/6a159d35426d452498e19d36602da178 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6a159d35426d452498e19d36602da178 2024-11-10T19:16:49,803 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6a159d35426d452498e19d36602da178, entries=150, sequenceid=147, filesize=30.4 K 2024-11-10T19:16:49,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/178aee35174b4726bcd9cf8a6c4b5b6a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/178aee35174b4726bcd9cf8a6c4b5b6a 2024-11-10T19:16:49,816 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/178aee35174b4726bcd9cf8a6c4b5b6a, entries=150, sequenceid=147, filesize=11.9 K 2024-11-10T19:16:49,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/fdd2098f8570482ca881ae90d23ecdf3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fdd2098f8570482ca881ae90d23ecdf3 2024-11-10T19:16:49,836 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fdd2098f8570482ca881ae90d23ecdf3, entries=150, sequenceid=147, filesize=11.9 K 2024-11-10T19:16:49,851 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 17e63e9bb9f2380be13ccc4351250608 in 1136ms, sequenceid=147, compaction requested=false 2024-11-10T19:16:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-10T19:16:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-10T19:16:49,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-10T19:16:49,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5350 sec 2024-11-10T19:16:49,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.5440 sec 2024-11-10T19:16:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:49,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T19:16:49,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:49,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:49,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:49,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:49,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:49,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:49,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110adcffdbada7d4169a30745903ea095b2_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266209547/Put/seqid=0 2024-11-10T19:16:50,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741980_1156 (size=14794) 2024-11-10T19:16:50,003 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:50,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110adcffdbada7d4169a30745903ea095b2_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110adcffdbada7d4169a30745903ea095b2_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:50,020 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/eb56978e4f0c4e9abee74f02fb08e161, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:50,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/eb56978e4f0c4e9abee74f02fb08e161 is 175, key is test_row_0/A:col10/1731266209547/Put/seqid=0 2024-11-10T19:16:50,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266270040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266270044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741981_1157 (size=39749) 2024-11-10T19:16:50,080 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/eb56978e4f0c4e9abee74f02fb08e161 2024-11-10T19:16:50,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/bc30fc514d9944edbaf75f0a10a3a93f is 50, key is test_row_0/B:col10/1731266209547/Put/seqid=0 2024-11-10T19:16:50,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741982_1158 (size=12151) 2024-11-10T19:16:50,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/bc30fc514d9944edbaf75f0a10a3a93f 2024-11-10T19:16:50,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/5aa2c8f2d8ad490dac0a7c0a897425b5 is 50, key is test_row_0/C:col10/1731266209547/Put/seqid=0 2024-11-10T19:16:50,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266270162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266270162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741983_1159 (size=12151) 2024-11-10T19:16:50,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266270372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266270372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/5aa2c8f2d8ad490dac0a7c0a897425b5 2024-11-10T19:16:50,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/eb56978e4f0c4e9abee74f02fb08e161 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/eb56978e4f0c4e9abee74f02fb08e161 2024-11-10T19:16:50,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/eb56978e4f0c4e9abee74f02fb08e161, entries=200, sequenceid=163, filesize=38.8 K 2024-11-10T19:16:50,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/bc30fc514d9944edbaf75f0a10a3a93f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bc30fc514d9944edbaf75f0a10a3a93f 2024-11-10T19:16:50,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bc30fc514d9944edbaf75f0a10a3a93f, entries=150, sequenceid=163, filesize=11.9 K 2024-11-10T19:16:50,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/5aa2c8f2d8ad490dac0a7c0a897425b5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5aa2c8f2d8ad490dac0a7c0a897425b5 2024-11-10T19:16:50,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5aa2c8f2d8ad490dac0a7c0a897425b5, entries=150, sequenceid=163, filesize=11.9 K 2024-11-10T19:16:50,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 17e63e9bb9f2380be13ccc4351250608 in 771ms, sequenceid=163, compaction requested=true 2024-11-10T19:16:50,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:50,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:50,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:50,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:50,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T19:16:50,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:50,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-10T19:16:50,681 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:50,681 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:50,683 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:50,683 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:16:50,684 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:50,684 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/33d04d31afe24c1f9a1cd47454a93004, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fdd2098f8570482ca881ae90d23ecdf3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5aa2c8f2d8ad490dac0a7c0a897425b5] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=35.7 K 2024-11-10T19:16:50,684 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:50,685 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:16:50,685 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:50,685 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/defd1d28a23140d3a24203f364326652, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6a159d35426d452498e19d36602da178, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/eb56978e4f0c4e9abee74f02fb08e161] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=99.6 K 2024-11-10T19:16:50,685 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:50,685 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/defd1d28a23140d3a24203f364326652, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6a159d35426d452498e19d36602da178, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/eb56978e4f0c4e9abee74f02fb08e161] 2024-11-10T19:16:50,688 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting defd1d28a23140d3a24203f364326652, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731266206592 2024-11-10T19:16:50,688 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d04d31afe24c1f9a1cd47454a93004, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731266206592 2024-11-10T19:16:50,690 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting fdd2098f8570482ca881ae90d23ecdf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731266207321 2024-11-10T19:16:50,690 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a159d35426d452498e19d36602da178, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731266207321 2024-11-10T19:16:50,691 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aa2c8f2d8ad490dac0a7c0a897425b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731266209538 2024-11-10T19:16:50,691 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb56978e4f0c4e9abee74f02fb08e161, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731266209529 2024-11-10T19:16:50,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T19:16:50,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:50,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:50,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:50,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:50,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:50,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:50,713 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:50,724 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411109b241c99b23c494b8314e036aa4a3e25_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:50,728 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411109b241c99b23c494b8314e036aa4a3e25_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:50,728 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109b241c99b23c494b8314e036aa4a3e25_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:50,734 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:50,735 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/357739ee54cf489c8e1a402cab01bc25 is 50, key is test_row_0/C:col10/1731266209547/Put/seqid=0 2024-11-10T19:16:50,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741984_1160 (size=4469) 2024-11-10T19:16:50,769 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#135 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:50,769 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/79c08f208aac40e3acf0e857a9b19dc2 is 175, key is test_row_0/A:col10/1731266209547/Put/seqid=0 2024-11-10T19:16:50,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266270758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110407e9dd220964d3eb363903426838f7c_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266210038/Put/seqid=0 2024-11-10T19:16:50,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266270767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741985_1161 (size=12459) 2024-11-10T19:16:50,837 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/357739ee54cf489c8e1a402cab01bc25 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/357739ee54cf489c8e1a402cab01bc25 2024-11-10T19:16:50,844 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into 357739ee54cf489c8e1a402cab01bc25(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:50,846 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:50,846 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266210680; duration=0sec 2024-11-10T19:16:50,846 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:50,846 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:16:50,846 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:50,852 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:50,852 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:16:50,852 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:50,852 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/7fb5a54eb14245bd9309c3c5a067f340, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/178aee35174b4726bcd9cf8a6c4b5b6a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bc30fc514d9944edbaf75f0a10a3a93f] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=35.7 K 2024-11-10T19:16:50,853 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fb5a54eb14245bd9309c3c5a067f340, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731266206592 2024-11-10T19:16:50,853 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 178aee35174b4726bcd9cf8a6c4b5b6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731266207321 2024-11-10T19:16:50,854 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting bc30fc514d9944edbaf75f0a10a3a93f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731266209538 2024-11-10T19:16:50,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741986_1162 (size=31413) 2024-11-10T19:16:50,882 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/79c08f208aac40e3acf0e857a9b19dc2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/79c08f208aac40e3acf0e857a9b19dc2 2024-11-10T19:16:50,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266270881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741987_1163 (size=14794) 2024-11-10T19:16:50,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:50,891 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into 79c08f208aac40e3acf0e857a9b19dc2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:50,892 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:50,892 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266210680; duration=0sec 2024-11-10T19:16:50,892 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:50,892 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:16:50,898 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110407e9dd220964d3eb363903426838f7c_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110407e9dd220964d3eb363903426838f7c_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:50,898 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#138 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:50,899 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/fee0aee56fae433d9aee600958a73502 is 50, key is test_row_0/B:col10/1731266209547/Put/seqid=0 2024-11-10T19:16:50,901 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8a0d75fefc154b0f879a25ba29c092b9, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:50,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8a0d75fefc154b0f879a25ba29c092b9 is 175, key is test_row_0/A:col10/1731266210038/Put/seqid=0 2024-11-10T19:16:50,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266270906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:50,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741988_1164 (size=12459) 2024-11-10T19:16:50,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741989_1165 (size=39749) 2024-11-10T19:16:50,969 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/fee0aee56fae433d9aee600958a73502 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fee0aee56fae433d9aee600958a73502 2024-11-10T19:16:50,992 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into fee0aee56fae433d9aee600958a73502(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:50,992 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:50,992 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266210680; duration=0sec 2024-11-10T19:16:50,992 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:50,992 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:16:51,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266271091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:51,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266271122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:51,370 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8a0d75fefc154b0f879a25ba29c092b9 2024-11-10T19:16:51,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/91faf3f6c62546af9c2b0cb9c316d263 is 50, key is test_row_0/B:col10/1731266210038/Put/seqid=0 2024-11-10T19:16:51,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266271402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:51,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741990_1166 (size=12151) 2024-11-10T19:16:51,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266271451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T19:16:51,482 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:51,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-10T19:16:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T19:16:51,487 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:51,488 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:51,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T19:16:51,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T19:16:51,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:51,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:51,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:51,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:51,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:51,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T19:16:51,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:51,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:51,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:51,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:51,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T19:16:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:51,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/91faf3f6c62546af9c2b0cb9c316d263 2024-11-10T19:16:51,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/189a76eae56e4be6a33bcb7b990efa23 is 50, key is test_row_0/C:col10/1731266210038/Put/seqid=0 2024-11-10T19:16:51,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741991_1167 (size=12151) 2024-11-10T19:16:51,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/189a76eae56e4be6a33bcb7b990efa23 2024-11-10T19:16:51,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8a0d75fefc154b0f879a25ba29c092b9 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a0d75fefc154b0f879a25ba29c092b9 2024-11-10T19:16:51,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a0d75fefc154b0f879a25ba29c092b9, entries=200, sequenceid=183, filesize=38.8 K 2024-11-10T19:16:51,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/91faf3f6c62546af9c2b0cb9c316d263 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91faf3f6c62546af9c2b0cb9c316d263 2024-11-10T19:16:51,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91faf3f6c62546af9c2b0cb9c316d263, entries=150, sequenceid=183, filesize=11.9 K 2024-11-10T19:16:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/189a76eae56e4be6a33bcb7b990efa23 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/189a76eae56e4be6a33bcb7b990efa23 2024-11-10T19:16:51,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/189a76eae56e4be6a33bcb7b990efa23, entries=150, sequenceid=183, filesize=11.9 K 2024-11-10T19:16:51,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 17e63e9bb9f2380be13ccc4351250608 in 1184ms, sequenceid=183, compaction requested=false 2024-11-10T19:16:51,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:51,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T19:16:51,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:51,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:51,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:51,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:51,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:51,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:51,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a09cdb09f5b94677b98027394507dac1_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266211912/Put/seqid=0 2024-11-10T19:16:51,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741992_1168 (size=14794) 2024-11-10T19:16:51,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T19:16:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:51,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:51,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:51,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266271980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266271990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:52,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266272101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266272101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T19:16:52,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T19:16:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T19:16:52,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266272311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266272311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:52,328 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:52,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a09cdb09f5b94677b98027394507dac1_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a09cdb09f5b94677b98027394507dac1_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:52,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/00b9a121e6ea45699f4a6279c150cb25, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:52,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/00b9a121e6ea45699f4a6279c150cb25 is 175, key is test_row_0/A:col10/1731266211912/Put/seqid=0 2024-11-10T19:16:52,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741993_1169 (size=39749) 2024-11-10T19:16:52,338 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/00b9a121e6ea45699f4a6279c150cb25 2024-11-10T19:16:52,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/e5374c291fe746ea881baf70d429f11b is 50, key is test_row_0/B:col10/1731266211912/Put/seqid=0 2024-11-10T19:16:52,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741994_1170 (size=12151) 2024-11-10T19:16:52,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/e5374c291fe746ea881baf70d429f11b 2024-11-10T19:16:52,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/383f5edfeea34720b65619572486a4d8 is 50, key is test_row_0/C:col10/1731266211912/Put/seqid=0 2024-11-10T19:16:52,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741995_1171 (size=12151) 2024-11-10T19:16:52,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T19:16:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,475 INFO [master/bdc25462c9ef:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T19:16:52,475 INFO [master/bdc25462c9ef:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T19:16:52,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T19:16:52,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:52,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T19:16:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266272621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266272621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:52,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T19:16:52,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:52,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:52,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/383f5edfeea34720b65619572486a4d8 2024-11-10T19:16:52,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/00b9a121e6ea45699f4a6279c150cb25 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/00b9a121e6ea45699f4a6279c150cb25 2024-11-10T19:16:52,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/00b9a121e6ea45699f4a6279c150cb25, entries=200, sequenceid=199, filesize=38.8 K 2024-11-10T19:16:52,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/e5374c291fe746ea881baf70d429f11b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e5374c291fe746ea881baf70d429f11b 2024-11-10T19:16:52,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e5374c291fe746ea881baf70d429f11b, entries=150, sequenceid=199, filesize=11.9 K 2024-11-10T19:16:52,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/383f5edfeea34720b65619572486a4d8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/383f5edfeea34720b65619572486a4d8 2024-11-10T19:16:52,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/383f5edfeea34720b65619572486a4d8, entries=150, sequenceid=199, filesize=11.9 K 2024-11-10T19:16:52,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 17e63e9bb9f2380be13ccc4351250608 in 932ms, sequenceid=199, compaction requested=true 2024-11-10T19:16:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:52,845 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:52,845 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:52,846 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:52,846 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:16:52,846 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:52,846 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:16:52,846 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,847 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,847 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/79c08f208aac40e3acf0e857a9b19dc2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a0d75fefc154b0f879a25ba29c092b9, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/00b9a121e6ea45699f4a6279c150cb25] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=108.3 K 2024-11-10T19:16:52,847 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fee0aee56fae433d9aee600958a73502, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91faf3f6c62546af9c2b0cb9c316d263, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e5374c291fe746ea881baf70d429f11b] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=35.9 K 2024-11-10T19:16:52,847 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,847 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/79c08f208aac40e3acf0e857a9b19dc2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a0d75fefc154b0f879a25ba29c092b9, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/00b9a121e6ea45699f4a6279c150cb25] 2024-11-10T19:16:52,847 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting fee0aee56fae433d9aee600958a73502, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731266209538 2024-11-10T19:16:52,847 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79c08f208aac40e3acf0e857a9b19dc2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731266209538 2024-11-10T19:16:52,847 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 91faf3f6c62546af9c2b0cb9c316d263, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731266210036 2024-11-10T19:16:52,848 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a0d75fefc154b0f879a25ba29c092b9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731266209970 2024-11-10T19:16:52,848 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting e5374c291fe746ea881baf70d429f11b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731266210735 2024-11-10T19:16:52,848 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b9a121e6ea45699f4a6279c150cb25, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731266210735 2024-11-10T19:16:52,857 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:52,868 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:52,869 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/ebf631d830c0481a9381d6aca42edb3d is 50, key is test_row_0/B:col10/1731266211912/Put/seqid=0 2024-11-10T19:16:52,871 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110dbe9a977db5140e5ba4f8e3203d2e370_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:52,873 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110dbe9a977db5140e5ba4f8e3203d2e370_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:52,874 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dbe9a977db5140e5ba4f8e3203d2e370_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:52,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T19:16:52,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:52,878 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T19:16:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:52,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741996_1172 (size=12561) 2024-11-10T19:16:52,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741997_1173 (size=4469) 2024-11-10T19:16:52,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b626cd0fdd8143e48bcccbd14dbe94c2_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266211985/Put/seqid=0 2024-11-10T19:16:52,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741998_1174 (size=12304) 2024-11-10T19:16:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:53,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266273162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266273163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266273272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266273272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,291 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/ebf631d830c0481a9381d6aca42edb3d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ebf631d830c0481a9381d6aca42edb3d 2024-11-10T19:16:53,292 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#144 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:53,293 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/9d1a1dd5152b4b5e97c1813d74560258 is 175, key is test_row_0/A:col10/1731266211912/Put/seqid=0 2024-11-10T19:16:53,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:53,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741999_1175 (size=31515) 2024-11-10T19:16:53,317 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b626cd0fdd8143e48bcccbd14dbe94c2_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b626cd0fdd8143e48bcccbd14dbe94c2_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:53,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/6e2907ee97c6437f9100a41157bdff27, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:53,318 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into ebf631d830c0481a9381d6aca42edb3d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:53,318 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:53,318 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266212845; duration=0sec 2024-11-10T19:16:53,319 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:53,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/6e2907ee97c6437f9100a41157bdff27 is 175, key is test_row_0/A:col10/1731266211985/Put/seqid=0 2024-11-10T19:16:53,319 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:16:53,319 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:53,322 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:53,322 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:16:53,322 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:53,322 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/357739ee54cf489c8e1a402cab01bc25, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/189a76eae56e4be6a33bcb7b990efa23, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/383f5edfeea34720b65619572486a4d8] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=35.9 K 2024-11-10T19:16:53,333 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 357739ee54cf489c8e1a402cab01bc25, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731266209538 2024-11-10T19:16:53,333 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/9d1a1dd5152b4b5e97c1813d74560258 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/9d1a1dd5152b4b5e97c1813d74560258 2024-11-10T19:16:53,336 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 189a76eae56e4be6a33bcb7b990efa23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731266210036 2024-11-10T19:16:53,337 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 383f5edfeea34720b65619572486a4d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731266210735 2024-11-10T19:16:53,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742000_1176 (size=31105) 2024-11-10T19:16:53,347 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/6e2907ee97c6437f9100a41157bdff27 2024-11-10T19:16:53,348 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into 9d1a1dd5152b4b5e97c1813d74560258(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:53,350 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:53,350 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266212845; duration=0sec 2024-11-10T19:16:53,350 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:53,350 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:16:53,351 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#147 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:53,352 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/f1ff957ff01e49ff97345b0d23432d87 is 50, key is test_row_0/C:col10/1731266211912/Put/seqid=0 2024-11-10T19:16:53,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/e4356593b91a43f0be290c6e5cae2f55 is 50, key is test_row_0/B:col10/1731266211985/Put/seqid=0 2024-11-10T19:16:53,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742002_1178 (size=12151) 2024-11-10T19:16:53,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742001_1177 (size=12561) 2024-11-10T19:16:53,379 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/e4356593b91a43f0be290c6e5cae2f55 2024-11-10T19:16:53,386 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/f1ff957ff01e49ff97345b0d23432d87 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/f1ff957ff01e49ff97345b0d23432d87 2024-11-10T19:16:53,397 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into f1ff957ff01e49ff97345b0d23432d87(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:53,397 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:53,398 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266212845; duration=0sec 2024-11-10T19:16:53,398 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:53,398 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:16:53,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/a0c701b267ba4c4395874ced17462597 is 50, key is test_row_0/C:col10/1731266211985/Put/seqid=0 2024-11-10T19:16:53,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742003_1179 (size=12151) 2024-11-10T19:16:53,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266273482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266273482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T19:16:53,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266273701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,703 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266273711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,714 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266273712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,715 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:16:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266273791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266273801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:53,841 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/a0c701b267ba4c4395874ced17462597 2024-11-10T19:16:53,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/6e2907ee97c6437f9100a41157bdff27 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6e2907ee97c6437f9100a41157bdff27 2024-11-10T19:16:53,855 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6e2907ee97c6437f9100a41157bdff27, entries=150, sequenceid=218, filesize=30.4 K 2024-11-10T19:16:53,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/e4356593b91a43f0be290c6e5cae2f55 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e4356593b91a43f0be290c6e5cae2f55 2024-11-10T19:16:53,870 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e4356593b91a43f0be290c6e5cae2f55, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T19:16:53,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/a0c701b267ba4c4395874ced17462597 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a0c701b267ba4c4395874ced17462597 2024-11-10T19:16:53,881 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a0c701b267ba4c4395874ced17462597, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T19:16:53,882 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 17e63e9bb9f2380be13ccc4351250608 in 1004ms, sequenceid=218, compaction requested=false 2024-11-10T19:16:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-10T19:16:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-10T19:16:53,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-10T19:16:53,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3950 sec 2024-11-10T19:16:53,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.4060 sec 2024-11-10T19:16:54,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T19:16:54,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:54,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:54,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:54,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:54,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:54,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:54,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ab02c9798d3e402faf4a90dfd4bd2548_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266214305/Put/seqid=0 2024-11-10T19:16:54,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742004_1180 (size=12304) 2024-11-10T19:16:54,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:54,387 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ab02c9798d3e402faf4a90dfd4bd2548_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ab02c9798d3e402faf4a90dfd4bd2548_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:54,388 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/f7ff12ab6ce8476fa6367e7ae93ceed7, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:54,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/f7ff12ab6ce8476fa6367e7ae93ceed7 is 175, key is test_row_0/A:col10/1731266214305/Put/seqid=0 2024-11-10T19:16:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266274402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:54,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742005_1181 (size=31105) 2024-11-10T19:16:54,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266274408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:54,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266274513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:54,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266274521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:54,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266274728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:54,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266274731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:54,812 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/f7ff12ab6ce8476fa6367e7ae93ceed7 2024-11-10T19:16:54,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/247fd5e0be38428cbfaf909815e976c2 is 50, key is test_row_0/B:col10/1731266214305/Put/seqid=0 2024-11-10T19:16:54,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742006_1182 (size=12151) 2024-11-10T19:16:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266275031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:55,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266275041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:55,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/247fd5e0be38428cbfaf909815e976c2 2024-11-10T19:16:55,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/1b8184a4d73d4ec092dfda62c4aa388e is 50, key is test_row_0/C:col10/1731266214305/Put/seqid=0 2024-11-10T19:16:55,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742007_1183 (size=12151) 2024-11-10T19:16:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266275541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:55,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266275551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T19:16:55,652 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:55,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-10T19:16:55,654 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:55,656 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:55,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T19:16:55,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/1b8184a4d73d4ec092dfda62c4aa388e 2024-11-10T19:16:55,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/f7ff12ab6ce8476fa6367e7ae93ceed7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/f7ff12ab6ce8476fa6367e7ae93ceed7 2024-11-10T19:16:55,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/f7ff12ab6ce8476fa6367e7ae93ceed7, entries=150, sequenceid=235, filesize=30.4 K 2024-11-10T19:16:55,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/247fd5e0be38428cbfaf909815e976c2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/247fd5e0be38428cbfaf909815e976c2 2024-11-10T19:16:55,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/247fd5e0be38428cbfaf909815e976c2, entries=150, sequenceid=235, filesize=11.9 K 2024-11-10T19:16:55,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/1b8184a4d73d4ec092dfda62c4aa388e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1b8184a4d73d4ec092dfda62c4aa388e 2024-11-10T19:16:55,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1b8184a4d73d4ec092dfda62c4aa388e, entries=150, sequenceid=235, filesize=11.9 K 2024-11-10T19:16:55,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 17e63e9bb9f2380be13ccc4351250608 in 1406ms, sequenceid=235, compaction requested=true 2024-11-10T19:16:55,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:55,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:55,717 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:55,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:55,717 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:55,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:55,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:55,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:55,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:55,718 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:55,718 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:55,718 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:16:55,718 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:16:55,719 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:55,719 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ebf631d830c0481a9381d6aca42edb3d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e4356593b91a43f0be290c6e5cae2f55, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/247fd5e0be38428cbfaf909815e976c2] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.0 K 2024-11-10T19:16:55,719 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:55,719 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ebf631d830c0481a9381d6aca42edb3d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731266210735 2024-11-10T19:16:55,719 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/9d1a1dd5152b4b5e97c1813d74560258, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6e2907ee97c6437f9100a41157bdff27, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/f7ff12ab6ce8476fa6367e7ae93ceed7] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=91.5 K 2024-11-10T19:16:55,720 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:55,720 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/9d1a1dd5152b4b5e97c1813d74560258, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6e2907ee97c6437f9100a41157bdff27, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/f7ff12ab6ce8476fa6367e7ae93ceed7] 2024-11-10T19:16:55,720 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d1a1dd5152b4b5e97c1813d74560258, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731266210735 2024-11-10T19:16:55,720 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting e4356593b91a43f0be290c6e5cae2f55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731266211974 2024-11-10T19:16:55,721 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 247fd5e0be38428cbfaf909815e976c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731266213138 2024-11-10T19:16:55,722 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e2907ee97c6437f9100a41157bdff27, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731266211974 2024-11-10T19:16:55,722 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7ff12ab6ce8476fa6367e7ae93ceed7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731266213138 2024-11-10T19:16:55,731 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:55,743 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:55,743 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/6f91a02c9c1941be8fe713087900bce2 is 50, key is test_row_0/B:col10/1731266214305/Put/seqid=0 2024-11-10T19:16:55,753 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110209dc3b343144647b18c271b424d505a_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:55,755 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110209dc3b343144647b18c271b424d505a_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:55,755 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110209dc3b343144647b18c271b424d505a_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T19:16:55,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742008_1184 (size=12663) 2024-11-10T19:16:55,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T19:16:55,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:55,814 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T19:16:55,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:55,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:55,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:55,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:55,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:55,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:55,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742009_1185 (size=4469) 2024-11-10T19:16:55,827 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#153 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:55,828 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7c486ec64c8b472fa270688cd62b9f48 is 175, key is test_row_0/A:col10/1731266214305/Put/seqid=0 2024-11-10T19:16:55,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742010_1186 (size=31617) 2024-11-10T19:16:55,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104e80fb4d51f240669a7318b80ee70d1a_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266214398/Put/seqid=0 2024-11-10T19:16:55,912 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7c486ec64c8b472fa270688cd62b9f48 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7c486ec64c8b472fa270688cd62b9f48 2024-11-10T19:16:55,920 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into 7c486ec64c8b472fa270688cd62b9f48(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:55,920 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:55,920 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266215717; duration=0sec 2024-11-10T19:16:55,928 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:55,928 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:16:55,928 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:55,930 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:55,930 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:16:55,930 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:55,930 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/f1ff957ff01e49ff97345b0d23432d87, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a0c701b267ba4c4395874ced17462597, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1b8184a4d73d4ec092dfda62c4aa388e] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.0 K 2024-11-10T19:16:55,930 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1ff957ff01e49ff97345b0d23432d87, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731266210735 2024-11-10T19:16:55,931 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0c701b267ba4c4395874ced17462597, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731266211974 2024-11-10T19:16:55,931 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b8184a4d73d4ec092dfda62c4aa388e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731266213138 2024-11-10T19:16:55,953 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#156 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:55,953 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/b01592897a5e4e46a3291fb2f99f934a is 50, key is test_row_0/C:col10/1731266214305/Put/seqid=0 2024-11-10T19:16:55,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742011_1187 (size=12304) 2024-11-10T19:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T19:16:55,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742012_1188 (size=12663) 2024-11-10T19:16:56,215 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/6f91a02c9c1941be8fe713087900bce2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/6f91a02c9c1941be8fe713087900bce2 2024-11-10T19:16:56,221 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into 6f91a02c9c1941be8fe713087900bce2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:56,221 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:56,221 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266215717; duration=0sec 2024-11-10T19:16:56,222 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:56,222 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:16:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T19:16:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:56,362 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104e80fb4d51f240669a7318b80ee70d1a_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e80fb4d51f240669a7318b80ee70d1a_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:56,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/1b2e8a413291419ba97b4250ece1db28, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/1b2e8a413291419ba97b4250ece1db28 is 175, key is test_row_0/A:col10/1731266214398/Put/seqid=0 2024-11-10T19:16:56,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742013_1189 (size=31105) 2024-11-10T19:16:56,387 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/b01592897a5e4e46a3291fb2f99f934a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/b01592897a5e4e46a3291fb2f99f934a 2024-11-10T19:16:56,392 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into b01592897a5e4e46a3291fb2f99f934a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:56,392 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:56,392 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266215717; duration=0sec 2024-11-10T19:16:56,393 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:56,393 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:16:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:56,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266276614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:56,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266276617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:56,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266276732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:56,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266276732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:56,768 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/1b2e8a413291419ba97b4250ece1db28 2024-11-10T19:16:56,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/1b2de53ad8824925bc2f5c1003737ffd is 50, key is test_row_0/B:col10/1731266214398/Put/seqid=0 2024-11-10T19:16:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T19:16:56,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742014_1190 (size=12151) 2024-11-10T19:16:56,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266276941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:56,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266276942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:57,197 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/1b2de53ad8824925bc2f5c1003737ffd 2024-11-10T19:16:57,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/73ce0c8568674033b6051779733c9f09 is 50, key is test_row_0/C:col10/1731266214398/Put/seqid=0 2024-11-10T19:16:57,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742015_1191 (size=12151) 2024-11-10T19:16:57,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266277251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:57,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266277251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:57,616 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/73ce0c8568674033b6051779733c9f09 2024-11-10T19:16:57,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/1b2e8a413291419ba97b4250ece1db28 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/1b2e8a413291419ba97b4250ece1db28 2024-11-10T19:16:57,625 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/1b2e8a413291419ba97b4250ece1db28, entries=150, sequenceid=253, filesize=30.4 K 2024-11-10T19:16:57,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/1b2de53ad8824925bc2f5c1003737ffd as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1b2de53ad8824925bc2f5c1003737ffd 2024-11-10T19:16:57,631 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1b2de53ad8824925bc2f5c1003737ffd, entries=150, sequenceid=253, filesize=11.9 K 2024-11-10T19:16:57,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/73ce0c8568674033b6051779733c9f09 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/73ce0c8568674033b6051779733c9f09 2024-11-10T19:16:57,638 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/73ce0c8568674033b6051779733c9f09, entries=150, sequenceid=253, filesize=11.9 K 2024-11-10T19:16:57,639 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 17e63e9bb9f2380be13ccc4351250608 in 1826ms, sequenceid=253, compaction requested=false 2024-11-10T19:16:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-10T19:16:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-10T19:16:57,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-10T19:16:57,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9840 sec 2024-11-10T19:16:57,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.9900 sec 2024-11-10T19:16:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:57,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T19:16:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:57,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b798a983d4bf4ac3858d20a2934132d0_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266216595/Put/seqid=0 2024-11-10T19:16:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T19:16:57,802 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:57,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-10T19:16:57,805 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T19:16:57,806 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:57,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:57,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742016_1192 (size=14994) 2024-11-10T19:16:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266277815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:57,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266277817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T19:16:57,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:57,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266277931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:57,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:57,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266277931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:57,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T19:16:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:57,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:58,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T19:16:58,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:58,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:58,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:58,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:58,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T19:16:58,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266278142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:58,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266278142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:58,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:58,215 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b798a983d4bf4ac3858d20a2934132d0_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b798a983d4bf4ac3858d20a2934132d0_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:58,216 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/20972292b673487fbf30c9caada0717b, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:58,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/20972292b673487fbf30c9caada0717b is 175, key is test_row_0/A:col10/1731266216595/Put/seqid=0 2024-11-10T19:16:58,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742017_1193 (size=39949) 2024-11-10T19:16:58,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/20972292b673487fbf30c9caada0717b 2024-11-10T19:16:58,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/f22411b276d0462893e6cb4d2da0e6ee is 50, key is test_row_0/B:col10/1731266216595/Put/seqid=0 2024-11-10T19:16:58,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742018_1194 (size=12301) 2024-11-10T19:16:58,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/f22411b276d0462893e6cb4d2da0e6ee 2024-11-10T19:16:58,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T19:16:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:58,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:16:58,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/bab71479456749d79c73b8de28e0c0b1 is 50, key is test_row_0/C:col10/1731266216595/Put/seqid=0 2024-11-10T19:16:58,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742019_1195 (size=12301) 2024-11-10T19:16:58,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/bab71479456749d79c73b8de28e0c0b1 2024-11-10T19:16:58,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/20972292b673487fbf30c9caada0717b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/20972292b673487fbf30c9caada0717b 2024-11-10T19:16:58,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/20972292b673487fbf30c9caada0717b, entries=200, sequenceid=271, filesize=39.0 K 2024-11-10T19:16:58,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/f22411b276d0462893e6cb4d2da0e6ee as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f22411b276d0462893e6cb4d2da0e6ee 2024-11-10T19:16:58,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f22411b276d0462893e6cb4d2da0e6ee, entries=150, sequenceid=271, filesize=12.0 K 2024-11-10T19:16:58,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/bab71479456749d79c73b8de28e0c0b1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/bab71479456749d79c73b8de28e0c0b1 2024-11-10T19:16:58,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/bab71479456749d79c73b8de28e0c0b1, entries=150, sequenceid=271, filesize=12.0 K 2024-11-10T19:16:58,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 17e63e9bb9f2380be13ccc4351250608 in 621ms, sequenceid=271, compaction requested=true 2024-11-10T19:16:58,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:58,385 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:58,386 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:58,386 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:16:58,386 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:58,386 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7c486ec64c8b472fa270688cd62b9f48, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/1b2e8a413291419ba97b4250ece1db28, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/20972292b673487fbf30c9caada0717b] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=100.3 K 2024-11-10T19:16:58,387 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:58,387 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7c486ec64c8b472fa270688cd62b9f48, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/1b2e8a413291419ba97b4250ece1db28, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/20972292b673487fbf30c9caada0717b] 2024-11-10T19:16:58,387 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c486ec64c8b472fa270688cd62b9f48, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731266213138 2024-11-10T19:16:58,388 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b2e8a413291419ba97b4250ece1db28, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731266214390 2024-11-10T19:16:58,388 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20972292b673487fbf30c9caada0717b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731266216595 2024-11-10T19:16:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:58,394 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:58,396 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:58,396 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:16:58,396 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:58,396 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/6f91a02c9c1941be8fe713087900bce2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1b2de53ad8824925bc2f5c1003737ffd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f22411b276d0462893e6cb4d2da0e6ee] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.2 K 2024-11-10T19:16:58,396 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f91a02c9c1941be8fe713087900bce2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731266213138 2024-11-10T19:16:58,397 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b2de53ad8824925bc2f5c1003737ffd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731266214390 2024-11-10T19:16:58,397 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting f22411b276d0462893e6cb4d2da0e6ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731266216595 2024-11-10T19:16:58,405 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:58,411 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:58,411 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/1811c4ab001d4138aea59da7154781b7 is 50, key is test_row_0/B:col10/1731266216595/Put/seqid=0 2024-11-10T19:16:58,417 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110244a3148864a48a8a79b10b35ea611f7_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:58,419 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110244a3148864a48a8a79b10b35ea611f7_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:58,419 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110244a3148864a48a8a79b10b35ea611f7_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T19:16:58,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T19:16:58,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:58,442 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:58,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742020_1196 (size=12915) 2024-11-10T19:16:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:58,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:16:58,466 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/1811c4ab001d4138aea59da7154781b7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1811c4ab001d4138aea59da7154781b7 2024-11-10T19:16:58,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742021_1197 (size=4469) 2024-11-10T19:16:58,477 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into 1811c4ab001d4138aea59da7154781b7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:58,477 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:58,477 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266218394; duration=0sec 2024-11-10T19:16:58,478 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:58,478 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:16:58,478 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:58,480 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:58,480 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:16:58,480 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:58,481 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/b01592897a5e4e46a3291fb2f99f934a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/73ce0c8568674033b6051779733c9f09, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/bab71479456749d79c73b8de28e0c0b1] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.2 K 2024-11-10T19:16:58,482 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting b01592897a5e4e46a3291fb2f99f934a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731266213138 2024-11-10T19:16:58,483 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ce0c8568674033b6051779733c9f09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731266214390 2024-11-10T19:16:58,485 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting bab71479456749d79c73b8de28e0c0b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731266216595 2024-11-10T19:16:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dad15c3950f248aa9ed13881efe249e2_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266217815/Put/seqid=0 2024-11-10T19:16:58,507 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:58,507 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/228fe7cd284a47dbb34c459cee56f6fd is 50, key is test_row_0/C:col10/1731266216595/Put/seqid=0 2024-11-10T19:16:58,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266278503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266278504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:58,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742022_1198 (size=12454) 2024-11-10T19:16:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:58,522 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dad15c3950f248aa9ed13881efe249e2_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dad15c3950f248aa9ed13881efe249e2_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:58,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/3044ada355754b6fb9abbbf0d4e130d8, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:58,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/3044ada355754b6fb9abbbf0d4e130d8 is 175, key is test_row_0/A:col10/1731266217815/Put/seqid=0 2024-11-10T19:16:58,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742023_1199 (size=12915) 2024-11-10T19:16:58,567 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/228fe7cd284a47dbb34c459cee56f6fd as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/228fe7cd284a47dbb34c459cee56f6fd 2024-11-10T19:16:58,577 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into 228fe7cd284a47dbb34c459cee56f6fd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:58,577 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:58,577 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266218394; duration=0sec 2024-11-10T19:16:58,577 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:58,577 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:16:58,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742024_1200 (size=31255) 2024-11-10T19:16:58,589 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/3044ada355754b6fb9abbbf0d4e130d8 2024-11-10T19:16:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/ea03c2e063764445a6b5f1a928132778 is 50, key is test_row_0/B:col10/1731266217815/Put/seqid=0 2024-11-10T19:16:58,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742025_1201 (size=12301) 2024-11-10T19:16:58,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266278621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:58,628 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/ea03c2e063764445a6b5f1a928132778 2024-11-10T19:16:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266278621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:58,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/9b3d09b40c754479b7a4b9eefb1beb5c is 50, key is test_row_0/C:col10/1731266217815/Put/seqid=0 2024-11-10T19:16:58,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742026_1202 (size=12301) 2024-11-10T19:16:58,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266278831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266278841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:58,871 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#162 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:58,872 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/b0578383a00e42dc8783c52b97c8ddf0 is 175, key is test_row_0/A:col10/1731266216595/Put/seqid=0 2024-11-10T19:16:58,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742027_1203 (size=31869) 2024-11-10T19:16:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T19:16:59,087 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/9b3d09b40c754479b7a4b9eefb1beb5c 2024-11-10T19:16:59,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/3044ada355754b6fb9abbbf0d4e130d8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/3044ada355754b6fb9abbbf0d4e130d8 2024-11-10T19:16:59,096 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/3044ada355754b6fb9abbbf0d4e130d8, entries=150, sequenceid=288, filesize=30.5 K 2024-11-10T19:16:59,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/ea03c2e063764445a6b5f1a928132778 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ea03c2e063764445a6b5f1a928132778 2024-11-10T19:16:59,102 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ea03c2e063764445a6b5f1a928132778, entries=150, sequenceid=288, filesize=12.0 K 2024-11-10T19:16:59,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/9b3d09b40c754479b7a4b9eefb1beb5c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/9b3d09b40c754479b7a4b9eefb1beb5c 2024-11-10T19:16:59,107 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/9b3d09b40c754479b7a4b9eefb1beb5c, entries=150, sequenceid=288, filesize=12.0 K 2024-11-10T19:16:59,108 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 17e63e9bb9f2380be13ccc4351250608 in 666ms, sequenceid=288, compaction requested=false 2024-11-10T19:16:59,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:59,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:59,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-10T19:16:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-10T19:16:59,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-10T19:16:59,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3040 sec 2024-11-10T19:16:59,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3100 sec 2024-11-10T19:16:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:59,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:16:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:59,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:59,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:59,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111085b9609aabcb42dab19a58a84517c1d4_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266218500/Put/seqid=0 2024-11-10T19:16:59,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266279218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266279218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:59,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742028_1204 (size=14994) 2024-11-10T19:16:59,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:59,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111085b9609aabcb42dab19a58a84517c1d4_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111085b9609aabcb42dab19a58a84517c1d4_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:59,249 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7e09dd8784344282861ab051c74cd0c0, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:59,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7e09dd8784344282861ab051c74cd0c0 is 175, key is test_row_0/A:col10/1731266218500/Put/seqid=0 2024-11-10T19:16:59,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742029_1205 (size=39949) 2024-11-10T19:16:59,258 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7e09dd8784344282861ab051c74cd0c0 2024-11-10T19:16:59,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/c74f7a170f2c4760a962694811d5b84f is 50, key is test_row_0/B:col10/1731266218500/Put/seqid=0 2024-11-10T19:16:59,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742030_1206 (size=12301) 2024-11-10T19:16:59,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/c74f7a170f2c4760a962694811d5b84f 2024-11-10T19:16:59,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/0bb664f56c214553b6302c9b1959b851 is 50, key is test_row_0/C:col10/1731266218500/Put/seqid=0 2024-11-10T19:16:59,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742031_1207 (size=12301) 2024-11-10T19:16:59,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/0bb664f56c214553b6302c9b1959b851 2024-11-10T19:16:59,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/7e09dd8784344282861ab051c74cd0c0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7e09dd8784344282861ab051c74cd0c0 2024-11-10T19:16:59,300 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/b0578383a00e42dc8783c52b97c8ddf0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/b0578383a00e42dc8783c52b97c8ddf0 2024-11-10T19:16:59,308 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into b0578383a00e42dc8783c52b97c8ddf0(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:59,308 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:59,308 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266218384; duration=0sec 2024-11-10T19:16:59,308 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:59,308 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:16:59,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7e09dd8784344282861ab051c74cd0c0, entries=200, sequenceid=307, filesize=39.0 K 2024-11-10T19:16:59,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/c74f7a170f2c4760a962694811d5b84f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c74f7a170f2c4760a962694811d5b84f 2024-11-10T19:16:59,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c74f7a170f2c4760a962694811d5b84f, entries=150, sequenceid=307, filesize=12.0 K 2024-11-10T19:16:59,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/0bb664f56c214553b6302c9b1959b851 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0bb664f56c214553b6302c9b1959b851 2024-11-10T19:16:59,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0bb664f56c214553b6302c9b1959b851, entries=150, sequenceid=307, filesize=12.0 K 2024-11-10T19:16:59,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 17e63e9bb9f2380be13ccc4351250608 in 177ms, sequenceid=307, compaction requested=true 2024-11-10T19:16:59,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:59,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:16:59,334 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:59,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:59,334 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:59,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:16:59,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:59,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:16:59,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:59,335 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:59,335 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:16:59,335 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:59,335 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/b0578383a00e42dc8783c52b97c8ddf0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/3044ada355754b6fb9abbbf0d4e130d8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7e09dd8784344282861ab051c74cd0c0] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=100.7 K 2024-11-10T19:16:59,335 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:59,335 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/b0578383a00e42dc8783c52b97c8ddf0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/3044ada355754b6fb9abbbf0d4e130d8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7e09dd8784344282861ab051c74cd0c0] 2024-11-10T19:16:59,336 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:59,336 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0578383a00e42dc8783c52b97c8ddf0, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731266216595 2024-11-10T19:16:59,336 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:16:59,336 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:59,336 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1811c4ab001d4138aea59da7154781b7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ea03c2e063764445a6b5f1a928132778, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c74f7a170f2c4760a962694811d5b84f] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.6 K 2024-11-10T19:16:59,337 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3044ada355754b6fb9abbbf0d4e130d8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731266217786 2024-11-10T19:16:59,337 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 1811c4ab001d4138aea59da7154781b7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731266216595 2024-11-10T19:16:59,337 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e09dd8784344282861ab051c74cd0c0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731266218500 2024-11-10T19:16:59,337 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ea03c2e063764445a6b5f1a928132778, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731266217786 2024-11-10T19:16:59,337 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c74f7a170f2c4760a962694811d5b84f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731266218500 2024-11-10T19:16:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:59,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:16:59,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:16:59,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:59,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:16:59,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:59,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:16:59,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:16:59,352 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:59,354 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:59,355 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/1dbadae4434745a8b9bd7b1cb4466fca is 50, key is test_row_0/B:col10/1731266218500/Put/seqid=0 2024-11-10T19:16:59,356 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110e443ff7a0cb94801b8523776617c268f_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:59,358 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110e443ff7a0cb94801b8523776617c268f_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:59,358 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e443ff7a0cb94801b8523776617c268f_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:59,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110612799affddc49efa24380749b82dee1_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266219338/Put/seqid=0 2024-11-10T19:16:59,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742033_1209 (size=13017) 2024-11-10T19:16:59,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742034_1210 (size=12454) 2024-11-10T19:16:59,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266279412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:59,423 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/1dbadae4434745a8b9bd7b1cb4466fca as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1dbadae4434745a8b9bd7b1cb4466fca 2024-11-10T19:16:59,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742032_1208 (size=4469) 2024-11-10T19:16:59,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266279419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:59,429 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:16:59,434 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into 1dbadae4434745a8b9bd7b1cb4466fca(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:59,434 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:59,434 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266219334; duration=0sec 2024-11-10T19:16:59,434 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:16:59,434 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:16:59,435 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:16:59,436 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110612799affddc49efa24380749b82dee1_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110612799affddc49efa24380749b82dee1_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:16:59,436 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:16:59,436 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:16:59,437 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:16:59,437 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/228fe7cd284a47dbb34c459cee56f6fd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/9b3d09b40c754479b7a4b9eefb1beb5c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0bb664f56c214553b6302c9b1959b851] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.6 K 2024-11-10T19:16:59,437 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8a403b1fb5f947588e3927fd0f97fd5a, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:16:59,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8a403b1fb5f947588e3927fd0f97fd5a is 175, key is test_row_0/A:col10/1731266219338/Put/seqid=0 2024-11-10T19:16:59,461 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 228fe7cd284a47dbb34c459cee56f6fd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731266216595 2024-11-10T19:16:59,462 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b3d09b40c754479b7a4b9eefb1beb5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731266217786 2024-11-10T19:16:59,462 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb664f56c214553b6302c9b1959b851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731266218500 2024-11-10T19:16:59,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742035_1211 (size=31255) 2024-11-10T19:16:59,493 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:59,494 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/575fef51136d4d32aabf7ab5403ef904 is 50, key is test_row_0/C:col10/1731266218500/Put/seqid=0 2024-11-10T19:16:59,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742036_1212 (size=13017) 2024-11-10T19:16:59,524 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/575fef51136d4d32aabf7ab5403ef904 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/575fef51136d4d32aabf7ab5403ef904 2024-11-10T19:16:59,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266279532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:59,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:59,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266279532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:59,557 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into 575fef51136d4d32aabf7ab5403ef904(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:59,557 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:59,557 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266219334; duration=0sec 2024-11-10T19:16:59,557 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:59,557 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:16:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266279743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:59,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:16:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266279742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:16:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:16:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:16:59,834 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#172 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:16:59,834 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/da39d722914c42c49be8b2a857389e21 is 175, key is test_row_0/A:col10/1731266218500/Put/seqid=0 2024-11-10T19:16:59,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742037_1213 (size=31971) 2024-11-10T19:16:59,867 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/da39d722914c42c49be8b2a857389e21 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/da39d722914c42c49be8b2a857389e21 2024-11-10T19:16:59,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8a403b1fb5f947588e3927fd0f97fd5a 2024-11-10T19:16:59,881 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into da39d722914c42c49be8b2a857389e21(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:16:59,881 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:16:59,881 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266219334; duration=0sec 2024-11-10T19:16:59,881 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:16:59,881 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:16:59,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/91e84283191545fa88899824d0d51bba is 50, key is test_row_0/B:col10/1731266219338/Put/seqid=0 2024-11-10T19:16:59,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742038_1214 (size=12301) 2024-11-10T19:16:59,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/91e84283191545fa88899824d0d51bba 2024-11-10T19:16:59,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/a7b931bd0b9b45c6a08e1e23f9067707 is 50, key is test_row_0/C:col10/1731266219338/Put/seqid=0 2024-11-10T19:16:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T19:16:59,957 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:16:59,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:16:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-10T19:16:59,960 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:16:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T19:16:59,963 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:16:59,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:16:59,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742039_1215 (size=12301) 2024-11-10T19:16:59,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/a7b931bd0b9b45c6a08e1e23f9067707 2024-11-10T19:17:00,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/8a403b1fb5f947588e3927fd0f97fd5a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a403b1fb5f947588e3927fd0f97fd5a 2024-11-10T19:17:00,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a403b1fb5f947588e3927fd0f97fd5a, entries=150, sequenceid=326, filesize=30.5 K 2024-11-10T19:17:00,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/91e84283191545fa88899824d0d51bba as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91e84283191545fa88899824d0d51bba 2024-11-10T19:17:00,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91e84283191545fa88899824d0d51bba, entries=150, sequenceid=326, filesize=12.0 K 2024-11-10T19:17:00,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/a7b931bd0b9b45c6a08e1e23f9067707 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a7b931bd0b9b45c6a08e1e23f9067707 2024-11-10T19:17:00,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a7b931bd0b9b45c6a08e1e23f9067707, entries=150, sequenceid=326, filesize=12.0 K 2024-11-10T19:17:00,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 17e63e9bb9f2380be13ccc4351250608 in 718ms, sequenceid=326, compaction requested=false 2024-11-10T19:17:00,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:00,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T19:17:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:17:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:17:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:17:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T19:17:00,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111033587a667fef4a36a4d55fb4e55398cd_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266220062/Put/seqid=0 2024-11-10T19:17:00,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:00,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:00,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742040_1216 (size=14994) 2024-11-10T19:17:00,143 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:00,154 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111033587a667fef4a36a4d55fb4e55398cd_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111033587a667fef4a36a4d55fb4e55398cd_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:00,155 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/482c5abeea804f69a8fd6729a949b030, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:00,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/482c5abeea804f69a8fd6729a949b030 is 175, key is test_row_0/A:col10/1731266220062/Put/seqid=0 2024-11-10T19:17:00,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266280169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:00,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742041_1217 (size=39949) 2024-11-10T19:17:00,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266280172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T19:17:00,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266280301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:00,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266280301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:00,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:00,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:00,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266280511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:00,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266280512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:00,590 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/482c5abeea804f69a8fd6729a949b030 2024-11-10T19:17:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T19:17:00,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/f2c73683d0aa48e3b210c0c25f5c2407 is 50, key is test_row_0/B:col10/1731266220062/Put/seqid=0 2024-11-10T19:17:00,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:00,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:00,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742042_1218 (size=12301) 2024-11-10T19:17:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:00,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:00,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266280821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:00,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266280821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:00,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:00,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:00,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:00,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/f2c73683d0aa48e3b210c0c25f5c2407 2024-11-10T19:17:01,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/819833a362c646cf9a8dcdab0c1c7ac5 is 50, key is test_row_0/C:col10/1731266220062/Put/seqid=0 2024-11-10T19:17:01,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742043_1219 (size=12301) 2024-11-10T19:17:01,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:01,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:01,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T19:17:01,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:01,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:01,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266281331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:01,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266281341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:01,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:01,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:01,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:01,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/819833a362c646cf9a8dcdab0c1c7ac5 2024-11-10T19:17:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/482c5abeea804f69a8fd6729a949b030 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/482c5abeea804f69a8fd6729a949b030 2024-11-10T19:17:01,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/482c5abeea804f69a8fd6729a949b030, entries=200, sequenceid=344, filesize=39.0 K 2024-11-10T19:17:01,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/f2c73683d0aa48e3b210c0c25f5c2407 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f2c73683d0aa48e3b210c0c25f5c2407 2024-11-10T19:17:01,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f2c73683d0aa48e3b210c0c25f5c2407, entries=150, sequenceid=344, filesize=12.0 K 2024-11-10T19:17:01,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/819833a362c646cf9a8dcdab0c1c7ac5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/819833a362c646cf9a8dcdab0c1c7ac5 2024-11-10T19:17:01,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/819833a362c646cf9a8dcdab0c1c7ac5, entries=150, sequenceid=344, filesize=12.0 K 2024-11-10T19:17:01,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 17e63e9bb9f2380be13ccc4351250608 in 1399ms, sequenceid=344, compaction requested=true 2024-11-10T19:17:01,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:01,468 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:01,469 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:01,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:01,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:01,470 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:17:01,470 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:01,470 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:01,470 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/da39d722914c42c49be8b2a857389e21, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a403b1fb5f947588e3927fd0f97fd5a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/482c5abeea804f69a8fd6729a949b030] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=100.8 K 2024-11-10T19:17:01,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:01,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:01,470 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:01,470 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/da39d722914c42c49be8b2a857389e21, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a403b1fb5f947588e3927fd0f97fd5a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/482c5abeea804f69a8fd6729a949b030] 2024-11-10T19:17:01,471 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting da39d722914c42c49be8b2a857389e21, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731266218500 2024-11-10T19:17:01,471 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a403b1fb5f947588e3927fd0f97fd5a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731266219215 2024-11-10T19:17:01,471 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:01,471 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:17:01,471 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,472 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1dbadae4434745a8b9bd7b1cb4466fca, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91e84283191545fa88899824d0d51bba, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f2c73683d0aa48e3b210c0c25f5c2407] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.7 K 2024-11-10T19:17:01,472 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dbadae4434745a8b9bd7b1cb4466fca, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731266218500 2024-11-10T19:17:01,472 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 482c5abeea804f69a8fd6729a949b030, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731266219392 2024-11-10T19:17:01,473 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 91e84283191545fa88899824d0d51bba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731266219215 2024-11-10T19:17:01,473 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c73683d0aa48e3b210c0c25f5c2407, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731266219392 2024-11-10T19:17:01,481 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:01,494 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:01,494 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/8e2aae6fa1ef45aea19b0f7ee4fa9ff2 is 50, key is test_row_0/B:col10/1731266220062/Put/seqid=0 2024-11-10T19:17:01,496 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411107425c70215f44b74a4b46fde415d7a9b_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:01,498 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411107425c70215f44b74a4b46fde415d7a9b_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:01,498 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107425c70215f44b74a4b46fde415d7a9b_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:01,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742044_1220 (size=13119) 2024-11-10T19:17:01,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742045_1221 (size=4469) 2024-11-10T19:17:01,515 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/8e2aae6fa1ef45aea19b0f7ee4fa9ff2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/8e2aae6fa1ef45aea19b0f7ee4fa9ff2 2024-11-10T19:17:01,515 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#180 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:01,516 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/853da594d6234517858ffb5379bda69a is 175, key is test_row_0/A:col10/1731266220062/Put/seqid=0 2024-11-10T19:17:01,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742046_1222 (size=32073) 2024-11-10T19:17:01,521 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into 8e2aae6fa1ef45aea19b0f7ee4fa9ff2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:01,521 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:01,521 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266221470; duration=0sec 2024-11-10T19:17:01,522 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:01,522 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:17:01,522 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:01,523 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:01,523 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:17:01,523 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,523 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/575fef51136d4d32aabf7ab5403ef904, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a7b931bd0b9b45c6a08e1e23f9067707, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/819833a362c646cf9a8dcdab0c1c7ac5] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.7 K 2024-11-10T19:17:01,527 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 575fef51136d4d32aabf7ab5403ef904, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731266218500 2024-11-10T19:17:01,528 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting a7b931bd0b9b45c6a08e1e23f9067707, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731266219215 2024-11-10T19:17:01,529 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 819833a362c646cf9a8dcdab0c1c7ac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731266219392 2024-11-10T19:17:01,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T19:17:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,533 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:17:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:17:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:17:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:17:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:01,537 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/853da594d6234517858ffb5379bda69a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/853da594d6234517858ffb5379bda69a 2024-11-10T19:17:01,544 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:01,545 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into 853da594d6234517858ffb5379bda69a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:01,545 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:01,545 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266221468; duration=0sec 2024-11-10T19:17:01,545 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:01,545 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:17:01,545 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/21a3bef38f1045378938953244eeafda is 50, key is test_row_0/C:col10/1731266220062/Put/seqid=0 2024-11-10T19:17:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104c6d7f20e1ac439f8ec955965512045f_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266220169/Put/seqid=0 2024-11-10T19:17:01,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742047_1223 (size=13119) 2024-11-10T19:17:01,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742048_1224 (size=12454) 2024-11-10T19:17:01,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,575 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104c6d7f20e1ac439f8ec955965512045f_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c6d7f20e1ac439f8ec955965512045f_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/2fe95b85db4e478188e4398bf73e58d4, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:01,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/2fe95b85db4e478188e4398bf73e58d4 is 175, key is test_row_0/A:col10/1731266220169/Put/seqid=0 2024-11-10T19:17:01,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742049_1225 (size=31255) 2024-11-10T19:17:01,601 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/2fe95b85db4e478188e4398bf73e58d4 2024-11-10T19:17:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/25d384101d6441eb8b33d5781c082c5e is 50, key is test_row_0/B:col10/1731266220169/Put/seqid=0 2024-11-10T19:17:01,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742050_1226 (size=12301) 2024-11-10T19:17:01,647 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/25d384101d6441eb8b33d5781c082c5e 2024-11-10T19:17:01,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/04d0f5af8ff04b8daa2849b2ba1a8f1b is 50, key is test_row_0/C:col10/1731266220169/Put/seqid=0 2024-11-10T19:17:01,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742051_1227 (size=12301) 2024-11-10T19:17:01,683 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/04d0f5af8ff04b8daa2849b2ba1a8f1b 2024-11-10T19:17:01,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/2fe95b85db4e478188e4398bf73e58d4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/2fe95b85db4e478188e4398bf73e58d4 2024-11-10T19:17:01,704 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/2fe95b85db4e478188e4398bf73e58d4, entries=150, sequenceid=362, filesize=30.5 K 2024-11-10T19:17:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/25d384101d6441eb8b33d5781c082c5e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/25d384101d6441eb8b33d5781c082c5e 2024-11-10T19:17:01,727 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/25d384101d6441eb8b33d5781c082c5e, entries=150, sequenceid=362, filesize=12.0 K 2024-11-10T19:17:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/04d0f5af8ff04b8daa2849b2ba1a8f1b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/04d0f5af8ff04b8daa2849b2ba1a8f1b 2024-11-10T19:17:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,733 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/04d0f5af8ff04b8daa2849b2ba1a8f1b, entries=150, sequenceid=362, filesize=12.0 K 2024-11-10T19:17:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,735 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 17e63e9bb9f2380be13ccc4351250608 in 201ms, sequenceid=362, compaction requested=false 2024-11-10T19:17:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-10T19:17:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-10T19:17:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-10T19:17:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7740 sec 2024-11-10T19:17:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7870 sec 2024-11-10T19:17:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,975 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/21a3bef38f1045378938953244eeafda as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/21a3bef38f1045378938953244eeafda 2024-11-10T19:17:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,981 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into 21a3bef38f1045378938953244eeafda(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:01,981 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:01,982 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266221470; duration=0sec 2024-11-10T19:17:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,982 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,982 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:17:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T19:17:02,111 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T19:17:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,115 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T19:17:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,116 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:02,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T19:17:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T19:17:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T19:17:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T19:17:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T19:17:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-10T19:17:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 162 msec 2024-11-10T19:17:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T19:17:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,432 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T19:17:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T19:17:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,435 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,436 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:02,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:02,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:17:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:17:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:02,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:17:02,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:02,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:17:02,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:02,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111039f0cbdf4e92495eac3865a7a3c6d783_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266222438/Put/seqid=0 2024-11-10T19:17:02,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742052_1228 (size=22618) 2024-11-10T19:17:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,483 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111039f0cbdf4e92495eac3865a7a3c6d783_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111039f0cbdf4e92495eac3865a7a3c6d783_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/99ea803d4dd74a4ba7862e9fd47f9930, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:02,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/99ea803d4dd74a4ba7862e9fd47f9930 is 175, key is test_row_0/A:col10/1731266222438/Put/seqid=0 2024-11-10T19:17:02,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742053_1229 (size=66023) 2024-11-10T19:17:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/99ea803d4dd74a4ba7862e9fd47f9930 2024-11-10T19:17:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/435e8569bdd145d4940d9545552fb675 is 50, key is test_row_0/B:col10/1731266222438/Put/seqid=0 2024-11-10T19:17:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731266222441, max timestamp=1731266222532Num uniques 859; 2024-11-10T19:17:02,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T19:17:02,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731266222441, max timestamp=1731266222532Num uniques 859; 2024-11-10T19:17:02,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T19:17:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:02,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731266222441, max timestamp=1731266222532Num uniques 859; 2024-11-10T19:17:02,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T19:17:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T19:17:02,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266282552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:02,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742054_1230 (size=17181) 2024-11-10T19:17:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:02,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266282558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:02,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:02,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T19:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:02,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:02,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266282661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:02,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266282671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:02,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T19:17:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:02,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:02,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T19:17:02,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266282872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:02,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266282881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:02,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T19:17:02,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:02,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:02,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:02,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:02,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/435e8569bdd145d4940d9545552fb675 2024-11-10T19:17:02,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/3bd300e30c30482886749a8477352d49 is 50, key is test_row_0/C:col10/1731266222438/Put/seqid=0 2024-11-10T19:17:03,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742055_1231 (size=12301) 2024-11-10T19:17:03,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T19:17:03,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:03,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:03,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T19:17:03,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266283182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:03,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266283201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:03,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T19:17:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:03,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:03,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T19:17:03,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:03,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:03,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:03,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/3bd300e30c30482886749a8477352d49 2024-11-10T19:17:03,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/99ea803d4dd74a4ba7862e9fd47f9930 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/99ea803d4dd74a4ba7862e9fd47f9930 2024-11-10T19:17:03,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/99ea803d4dd74a4ba7862e9fd47f9930, entries=350, sequenceid=375, filesize=64.5 K 2024-11-10T19:17:03,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/435e8569bdd145d4940d9545552fb675 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/435e8569bdd145d4940d9545552fb675 2024-11-10T19:17:03,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/435e8569bdd145d4940d9545552fb675, entries=250, sequenceid=375, filesize=16.8 K 2024-11-10T19:17:03,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/3bd300e30c30482886749a8477352d49 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/3bd300e30c30482886749a8477352d49 2024-11-10T19:17:03,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/3bd300e30c30482886749a8477352d49, entries=150, sequenceid=375, filesize=12.0 K 2024-11-10T19:17:03,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 17e63e9bb9f2380be13ccc4351250608 in 1005ms, sequenceid=375, compaction requested=true 2024-11-10T19:17:03,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:03,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:03,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:03,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:03,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:03,444 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:03,444 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:03,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:03,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:03,445 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:03,445 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129351 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:03,445 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:17:03,445 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:17:03,445 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,445 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,445 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/8e2aae6fa1ef45aea19b0f7ee4fa9ff2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/25d384101d6441eb8b33d5781c082c5e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/435e8569bdd145d4940d9545552fb675] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=41.6 K 2024-11-10T19:17:03,445 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/853da594d6234517858ffb5379bda69a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/2fe95b85db4e478188e4398bf73e58d4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/99ea803d4dd74a4ba7862e9fd47f9930] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=126.3 K 2024-11-10T19:17:03,445 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,445 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/853da594d6234517858ffb5379bda69a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/2fe95b85db4e478188e4398bf73e58d4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/99ea803d4dd74a4ba7862e9fd47f9930] 2024-11-10T19:17:03,445 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e2aae6fa1ef45aea19b0f7ee4fa9ff2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731266219392 2024-11-10T19:17:03,446 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 853da594d6234517858ffb5379bda69a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731266219392 2024-11-10T19:17:03,446 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d384101d6441eb8b33d5781c082c5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731266220109 2024-11-10T19:17:03,446 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fe95b85db4e478188e4398bf73e58d4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731266220109 2024-11-10T19:17:03,446 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 435e8569bdd145d4940d9545552fb675, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731266222408 2024-11-10T19:17:03,446 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99ea803d4dd74a4ba7862e9fd47f9930, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731266222387 2024-11-10T19:17:03,457 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:03,458 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:03,459 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/fa14a1ac064841c3a2d51a07b73d7d42 is 50, key is test_row_0/B:col10/1731266222438/Put/seqid=0 2024-11-10T19:17:03,467 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111044235bac1b1f439592ccd4687a643f50_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:03,469 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111044235bac1b1f439592ccd4687a643f50_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:03,470 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111044235bac1b1f439592ccd4687a643f50_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:03,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742056_1232 (size=13221) 2024-11-10T19:17:03,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742057_1233 (size=4469) 2024-11-10T19:17:03,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T19:17:03,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,514 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-10T19:17:03,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:17:03,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:03,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:17:03,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:03,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:17:03,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:03,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f14d878b48cf4fe2a9b7a9b218388056_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266222550/Put/seqid=0 2024-11-10T19:17:03,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742058_1234 (size=12454) 2024-11-10T19:17:03,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:03,546 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f14d878b48cf4fe2a9b7a9b218388056_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f14d878b48cf4fe2a9b7a9b218388056_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/54bf3d7847f64c9b916168171735c0fd, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/54bf3d7847f64c9b916168171735c0fd is 175, key is test_row_0/A:col10/1731266222550/Put/seqid=0 2024-11-10T19:17:03,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742059_1235 (size=31255) 2024-11-10T19:17:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T19:17:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:03,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. as already flushing 2024-11-10T19:17:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266283719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:03,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266283721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51006 deadline: 1731266283721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:03,725 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:03,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:03,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50940 deadline: 1731266283741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:03,745 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:03,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:03,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51014 deadline: 1731266283781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:03,784 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 17e63e9bb9f2380be13ccc4351250608 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18310 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:03,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50966 deadline: 1731266283831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:03,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50994 deadline: 1731266283831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:03,877 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/fa14a1ac064841c3a2d51a07b73d7d42 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fa14a1ac064841c3a2d51a07b73d7d42 2024-11-10T19:17:03,880 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#190 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:03,880 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/21b3e3c3bf5249979f3e04ee098dc2b7 is 175, key is test_row_0/A:col10/1731266222438/Put/seqid=0 2024-11-10T19:17:03,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742060_1236 (size=32175) 2024-11-10T19:17:03,888 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into fa14a1ac064841c3a2d51a07b73d7d42(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:03,888 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:03,889 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266223444; duration=0sec 2024-11-10T19:17:03,889 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:03,889 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:17:03,889 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:03,890 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:03,890 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:17:03,890 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:03,890 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/21a3bef38f1045378938953244eeafda, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/04d0f5af8ff04b8daa2849b2ba1a8f1b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/3bd300e30c30482886749a8477352d49] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.8 K 2024-11-10T19:17:03,891 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a3bef38f1045378938953244eeafda, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731266219392 2024-11-10T19:17:03,892 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d0f5af8ff04b8daa2849b2ba1a8f1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731266220109 2024-11-10T19:17:03,893 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bd300e30c30482886749a8477352d49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731266222426 2024-11-10T19:17:03,894 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/21b3e3c3bf5249979f3e04ee098dc2b7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/21b3e3c3bf5249979f3e04ee098dc2b7 2024-11-10T19:17:03,900 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:03,901 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into 21b3e3c3bf5249979f3e04ee098dc2b7(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:03,901 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:03,901 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266223444; duration=0sec 2024-11-10T19:17:03,901 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/7e5534f717674d84ae74a4e6c63ae6c2 is 50, key is test_row_0/C:col10/1731266222438/Put/seqid=0 2024-11-10T19:17:03,901 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:03,901 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:17:03,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742061_1237 (size=13221) 2024-11-10T19:17:03,955 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/54bf3d7847f64c9b916168171735c0fd 2024-11-10T19:17:03,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/bbc9fc702e30405485bcabe0f1de26be is 50, key is test_row_0/B:col10/1731266222550/Put/seqid=0 2024-11-10T19:17:03,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742062_1238 (size=12301) 2024-11-10T19:17:03,976 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/bbc9fc702e30405485bcabe0f1de26be 2024-11-10T19:17:03,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/fb805c1a6ab54938b2a8c892bdfc7ba2 is 50, key is test_row_0/C:col10/1731266222550/Put/seqid=0 2024-11-10T19:17:04,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742063_1239 (size=12301) 2024-11-10T19:17:04,013 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/fb805c1a6ab54938b2a8c892bdfc7ba2 2024-11-10T19:17:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/54bf3d7847f64c9b916168171735c0fd as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/54bf3d7847f64c9b916168171735c0fd 2024-11-10T19:17:04,026 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/54bf3d7847f64c9b916168171735c0fd, entries=150, sequenceid=401, filesize=30.5 K 2024-11-10T19:17:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/bbc9fc702e30405485bcabe0f1de26be as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bbc9fc702e30405485bcabe0f1de26be 2024-11-10T19:17:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,035 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bbc9fc702e30405485bcabe0f1de26be, entries=150, sequenceid=401, filesize=12.0 K 2024-11-10T19:17:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/fb805c1a6ab54938b2a8c892bdfc7ba2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fb805c1a6ab54938b2a8c892bdfc7ba2 2024-11-10T19:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,043 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fb805c1a6ab54938b2a8c892bdfc7ba2, entries=150, sequenceid=401, filesize=12.0 K 2024-11-10T19:17:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,045 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=46.96 KB/48090 for 17e63e9bb9f2380be13ccc4351250608 in 530ms, sequenceid=401, compaction requested=false 2024-11-10T19:17:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:04,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T19:17:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T19:17:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T19:17:04,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6110 sec 2024-11-10T19:17:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6180 sec 2024-11-10T19:17:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:04,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T19:17:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:17:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:04,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:17:04,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:04,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:17:04,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,083 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-10T19:17:04,083 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:04,083 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,083 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,085 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-10T19:17:04,085 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:04,085 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,085 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:04,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ebdadd5d819f4557835eacbb8493bc30_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266224071/Put/seqid=0 2024-11-10T19:17:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,086 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-10T19:17:04,087 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:04,087 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,087 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:04,087 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-10T19:17:04,087 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:04,087 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,087 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742064_1240 (size=14994) 2024-11-10T19:17:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,099 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-10T19:17:04,099 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:04,099 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,099 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,106 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ebdadd5d819f4557835eacbb8493bc30_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ebdadd5d819f4557835eacbb8493bc30_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:04,107 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/449c857f00d14a189891b800bcb4b049, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:04,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/449c857f00d14a189891b800bcb4b049 is 175, key is test_row_0/A:col10/1731266224071/Put/seqid=0 2024-11-10T19:17:04,109 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-10T19:17:04,109 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:04,109 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,109 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:04,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:04,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742065_1241 (size=39949) 2024-11-10T19:17:04,316 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/7e5534f717674d84ae74a4e6c63ae6c2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/7e5534f717674d84ae74a4e6c63ae6c2 2024-11-10T19:17:04,325 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into 7e5534f717674d84ae74a4e6c63ae6c2(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:04,325 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:04,325 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266223444; duration=0sec 2024-11-10T19:17:04,325 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:04,325 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:17:04,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/449c857f00d14a189891b800bcb4b049 2024-11-10T19:17:04,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/70097d1e4d034319ac77111b75f3d9cd is 50, key is test_row_0/B:col10/1731266224071/Put/seqid=0 2024-11-10T19:17:04,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742066_1242 (size=12301) 2024-11-10T19:17:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T19:17:04,581 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:04,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/70097d1e4d034319ac77111b75f3d9cd 2024-11-10T19:17:04,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/4898c9c9179441c3b4acd3d564f51ed0 is 50, key is test_row_0/C:col10/1731266224071/Put/seqid=0 2024-11-10T19:17:04,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742067_1243 (size=12301) 2024-11-10T19:17:04,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/4898c9c9179441c3b4acd3d564f51ed0 2024-11-10T19:17:04,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/449c857f00d14a189891b800bcb4b049 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/449c857f00d14a189891b800bcb4b049 2024-11-10T19:17:04,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/449c857f00d14a189891b800bcb4b049, entries=200, sequenceid=415, filesize=39.0 K 2024-11-10T19:17:04,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/70097d1e4d034319ac77111b75f3d9cd as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/70097d1e4d034319ac77111b75f3d9cd 2024-11-10T19:17:04,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/70097d1e4d034319ac77111b75f3d9cd, entries=150, sequenceid=415, filesize=12.0 K 2024-11-10T19:17:04,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/4898c9c9179441c3b4acd3d564f51ed0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/4898c9c9179441c3b4acd3d564f51ed0 2024-11-10T19:17:04,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/4898c9c9179441c3b4acd3d564f51ed0, entries=150, sequenceid=415, filesize=12.0 K 2024-11-10T19:17:04,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 17e63e9bb9f2380be13ccc4351250608 in 925ms, sequenceid=415, compaction requested=true 2024-11-10T19:17:04,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:04,997 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:04,998 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:04,998 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/A is initiating minor compaction (all files) 2024-11-10T19:17:04,998 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/A in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:04,998 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/21b3e3c3bf5249979f3e04ee098dc2b7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/54bf3d7847f64c9b916168171735c0fd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/449c857f00d14a189891b800bcb4b049] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=101.0 K 2024-11-10T19:17:04,998 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:04,998 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/21b3e3c3bf5249979f3e04ee098dc2b7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/54bf3d7847f64c9b916168171735c0fd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/449c857f00d14a189891b800bcb4b049] 2024-11-10T19:17:05,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:05,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:05,000 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:05,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:05,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:05,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17e63e9bb9f2380be13ccc4351250608:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:05,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:05,003 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b3e3c3bf5249979f3e04ee098dc2b7, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731266222426 2024-11-10T19:17:05,004 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54bf3d7847f64c9b916168171735c0fd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731266222550 2024-11-10T19:17:05,005 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449c857f00d14a189891b800bcb4b049, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731266223713 2024-11-10T19:17:05,005 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:05,005 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/B is initiating minor compaction (all files) 2024-11-10T19:17:05,005 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/B in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:05,006 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fa14a1ac064841c3a2d51a07b73d7d42, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bbc9fc702e30405485bcabe0f1de26be, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/70097d1e4d034319ac77111b75f3d9cd] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.9 K 2024-11-10T19:17:05,006 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting fa14a1ac064841c3a2d51a07b73d7d42, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731266222426 2024-11-10T19:17:05,007 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc9fc702e30405485bcabe0f1de26be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731266222550 2024-11-10T19:17:05,007 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 70097d1e4d034319ac77111b75f3d9cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731266223713 2024-11-10T19:17:05,038 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#B#compaction#198 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:05,040 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/2475427f286442b59dc261a1f74237aa is 50, key is test_row_0/B:col10/1731266224071/Put/seqid=0 2024-11-10T19:17:05,054 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:05,061 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411104eb5a508004c49518cfe42630646494f_17e63e9bb9f2380be13ccc4351250608 store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:05,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742068_1244 (size=13323) 2024-11-10T19:17:05,089 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411104eb5a508004c49518cfe42630646494f_17e63e9bb9f2380be13ccc4351250608, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:05,089 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104eb5a508004c49518cfe42630646494f_17e63e9bb9f2380be13ccc4351250608 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:05,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742069_1245 (size=4469) 2024-11-10T19:17:05,144 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#A#compaction#199 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:05,145 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/0c25063d1b7d4618a1e9df1bbc2f1edd is 175, key is test_row_0/A:col10/1731266224071/Put/seqid=0 2024-11-10T19:17:05,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742070_1246 (size=32277) 2024-11-10T19:17:05,176 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/0c25063d1b7d4618a1e9df1bbc2f1edd as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0c25063d1b7d4618a1e9df1bbc2f1edd 2024-11-10T19:17:05,185 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/A of 17e63e9bb9f2380be13ccc4351250608 into 0c25063d1b7d4618a1e9df1bbc2f1edd(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:05,185 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:05,185 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/A, priority=13, startTime=1731266224996; duration=0sec 2024-11-10T19:17:05,185 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:05,185 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:A 2024-11-10T19:17:05,185 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:05,187 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:05,187 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17e63e9bb9f2380be13ccc4351250608/C is initiating minor compaction (all files) 2024-11-10T19:17:05,188 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17e63e9bb9f2380be13ccc4351250608/C in TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:05,188 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/7e5534f717674d84ae74a4e6c63ae6c2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fb805c1a6ab54938b2a8c892bdfc7ba2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/4898c9c9179441c3b4acd3d564f51ed0] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp, totalSize=36.9 K 2024-11-10T19:17:05,188 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e5534f717674d84ae74a4e6c63ae6c2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731266222426 2024-11-10T19:17:05,189 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb805c1a6ab54938b2a8c892bdfc7ba2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731266222550 2024-11-10T19:17:05,189 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4898c9c9179441c3b4acd3d564f51ed0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731266223713 2024-11-10T19:17:05,213 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17e63e9bb9f2380be13ccc4351250608#C#compaction#200 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:05,213 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/5cb71c55c27c4207a7092477c46fc846 is 50, key is test_row_0/C:col10/1731266224071/Put/seqid=0 2024-11-10T19:17:05,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742071_1247 (size=13323) 2024-11-10T19:17:05,488 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/2475427f286442b59dc261a1f74237aa as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/2475427f286442b59dc261a1f74237aa 2024-11-10T19:17:05,504 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/B of 17e63e9bb9f2380be13ccc4351250608 into 2475427f286442b59dc261a1f74237aa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:05,504 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:05,504 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/B, priority=13, startTime=1731266225000; duration=0sec 2024-11-10T19:17:05,504 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:05,504 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:B 2024-11-10T19:17:05,656 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/5cb71c55c27c4207a7092477c46fc846 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5cb71c55c27c4207a7092477c46fc846 2024-11-10T19:17:05,663 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17e63e9bb9f2380be13ccc4351250608/C of 17e63e9bb9f2380be13ccc4351250608 into 5cb71c55c27c4207a7092477c46fc846(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:05,663 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17e63e9bb9f2380be13ccc4351250608: 2024-11-10T19:17:05,663 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., storeName=17e63e9bb9f2380be13ccc4351250608/C, priority=13, startTime=1731266225000; duration=0sec 2024-11-10T19:17:05,664 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:05,664 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17e63e9bb9f2380be13ccc4351250608:C 2024-11-10T19:17:05,897 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T19:17:13,783 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-10T19:17:13,783 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:13,783 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:13,783 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:13,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:13,824 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-10T19:17:13,824 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:13,824 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:13,824 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:13,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:13,834 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-10T19:17:13,834 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:13,834 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:13,834 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:13,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2273 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2268 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1024 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3072 rows 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1040 2024-11-10T19:17:13,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3118 rows 2024-11-10T19:17:13,835 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:17:13,835 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:17:13,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:13,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:13,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:13,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T19:17:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T19:17:13,843 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266233843"}]},"ts":"1731266233843"} 2024-11-10T19:17:13,845 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T19:17:13,845 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T19:17:13,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T19:17:13,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, UNASSIGN}] 2024-11-10T19:17:13,850 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, UNASSIGN 2024-11-10T19:17:13,851 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=17e63e9bb9f2380be13ccc4351250608, regionState=CLOSING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:13,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, UNASSIGN because future has completed 2024-11-10T19:17:13,853 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T19:17:13,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:17:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T19:17:14,006 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:14,006 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T19:17:14,007 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing 17e63e9bb9f2380be13ccc4351250608, disabling compactions & flushes 2024-11-10T19:17:14,007 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:14,007 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:14,007 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. after waiting 0 ms 2024-11-10T19:17:14,007 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:14,007 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing 17e63e9bb9f2380be13ccc4351250608 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T19:17:14,007 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=A 2024-11-10T19:17:14,008 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:14,008 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=B 2024-11-10T19:17:14,008 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:14,008 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17e63e9bb9f2380be13ccc4351250608, store=C 2024-11-10T19:17:14,008 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:14,017 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e32bb5d477d74b76a88385e0314981af_17e63e9bb9f2380be13ccc4351250608 is 50, key is test_row_0/A:col10/1731266233831/Put/seqid=0 2024-11-10T19:17:14,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742072_1248 (size=9914) 2024-11-10T19:17:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T19:17:14,423 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:14,433 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e32bb5d477d74b76a88385e0314981af_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e32bb5d477d74b76a88385e0314981af_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:14,434 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/206a72fdb79b48aaa77dd9f966ff85dc, store: [table=TestAcidGuarantees family=A region=17e63e9bb9f2380be13ccc4351250608] 2024-11-10T19:17:14,435 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/206a72fdb79b48aaa77dd9f966ff85dc is 175, key is test_row_0/A:col10/1731266233831/Put/seqid=0 2024-11-10T19:17:14,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742073_1249 (size=22561) 2024-11-10T19:17:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T19:17:14,839 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/206a72fdb79b48aaa77dd9f966ff85dc 2024-11-10T19:17:14,846 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/fee7b98088c349828df1a8e527679803 is 50, key is test_row_0/B:col10/1731266233831/Put/seqid=0 2024-11-10T19:17:14,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742074_1250 (size=9857) 2024-11-10T19:17:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T19:17:15,250 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/fee7b98088c349828df1a8e527679803 2024-11-10T19:17:15,260 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/aa444bf5620a4bada8483444f6752e7f is 50, key is test_row_0/C:col10/1731266233831/Put/seqid=0 2024-11-10T19:17:15,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742075_1251 (size=9857) 2024-11-10T19:17:15,674 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/aa444bf5620a4bada8483444f6752e7f 2024-11-10T19:17:15,686 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/A/206a72fdb79b48aaa77dd9f966ff85dc as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/206a72fdb79b48aaa77dd9f966ff85dc 2024-11-10T19:17:15,707 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/206a72fdb79b48aaa77dd9f966ff85dc, entries=100, sequenceid=428, filesize=22.0 K 2024-11-10T19:17:15,708 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/B/fee7b98088c349828df1a8e527679803 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fee7b98088c349828df1a8e527679803 2024-11-10T19:17:15,715 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fee7b98088c349828df1a8e527679803, entries=100, sequenceid=428, filesize=9.6 K 2024-11-10T19:17:15,716 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/.tmp/C/aa444bf5620a4bada8483444f6752e7f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/aa444bf5620a4bada8483444f6752e7f 2024-11-10T19:17:15,722 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/aa444bf5620a4bada8483444f6752e7f, entries=100, sequenceid=428, filesize=9.6 K 2024-11-10T19:17:15,724 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 17e63e9bb9f2380be13ccc4351250608 in 1717ms, sequenceid=428, compaction requested=false 2024-11-10T19:17:15,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/bf4c65acd6b741ac8d3f9513507f94b8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/ef439ec78a114bdeb19e3af194478309, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0eaf46fded8948cca2f2fd1051e74445, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8bdc231c4fa04ad8a104bf47024810da, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7a22be4965bf48b99429831cf4aa3e71, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/defd1d28a23140d3a24203f364326652, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6a159d35426d452498e19d36602da178, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/eb56978e4f0c4e9abee74f02fb08e161, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/79c08f208aac40e3acf0e857a9b19dc2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a0d75fefc154b0f879a25ba29c092b9, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/00b9a121e6ea45699f4a6279c150cb25, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/9d1a1dd5152b4b5e97c1813d74560258, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6e2907ee97c6437f9100a41157bdff27, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7c486ec64c8b472fa270688cd62b9f48, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/f7ff12ab6ce8476fa6367e7ae93ceed7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/1b2e8a413291419ba97b4250ece1db28, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/20972292b673487fbf30c9caada0717b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/b0578383a00e42dc8783c52b97c8ddf0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/3044ada355754b6fb9abbbf0d4e130d8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7e09dd8784344282861ab051c74cd0c0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/da39d722914c42c49be8b2a857389e21, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a403b1fb5f947588e3927fd0f97fd5a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/482c5abeea804f69a8fd6729a949b030, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/853da594d6234517858ffb5379bda69a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/2fe95b85db4e478188e4398bf73e58d4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/99ea803d4dd74a4ba7862e9fd47f9930, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/21b3e3c3bf5249979f3e04ee098dc2b7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/54bf3d7847f64c9b916168171735c0fd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/449c857f00d14a189891b800bcb4b049] to archive 2024-11-10T19:17:15,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:17:15,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/bf4c65acd6b741ac8d3f9513507f94b8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/bf4c65acd6b741ac8d3f9513507f94b8 2024-11-10T19:17:15,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0bc7ecb3a13b4b01b1bb70c8b9861e4b 2024-11-10T19:17:15,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/ef439ec78a114bdeb19e3af194478309 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/ef439ec78a114bdeb19e3af194478309 2024-11-10T19:17:15,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0eaf46fded8948cca2f2fd1051e74445 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0eaf46fded8948cca2f2fd1051e74445 2024-11-10T19:17:15,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8bdc231c4fa04ad8a104bf47024810da to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8bdc231c4fa04ad8a104bf47024810da 2024-11-10T19:17:15,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7a22be4965bf48b99429831cf4aa3e71 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7a22be4965bf48b99429831cf4aa3e71 2024-11-10T19:17:15,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/defd1d28a23140d3a24203f364326652 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/defd1d28a23140d3a24203f364326652 2024-11-10T19:17:15,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6a159d35426d452498e19d36602da178 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6a159d35426d452498e19d36602da178 2024-11-10T19:17:15,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/eb56978e4f0c4e9abee74f02fb08e161 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/eb56978e4f0c4e9abee74f02fb08e161 2024-11-10T19:17:15,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/79c08f208aac40e3acf0e857a9b19dc2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/79c08f208aac40e3acf0e857a9b19dc2 2024-11-10T19:17:15,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a0d75fefc154b0f879a25ba29c092b9 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a0d75fefc154b0f879a25ba29c092b9 2024-11-10T19:17:15,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/00b9a121e6ea45699f4a6279c150cb25 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/00b9a121e6ea45699f4a6279c150cb25 2024-11-10T19:17:15,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/9d1a1dd5152b4b5e97c1813d74560258 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/9d1a1dd5152b4b5e97c1813d74560258 2024-11-10T19:17:15,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6e2907ee97c6437f9100a41157bdff27 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/6e2907ee97c6437f9100a41157bdff27 2024-11-10T19:17:15,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7c486ec64c8b472fa270688cd62b9f48 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7c486ec64c8b472fa270688cd62b9f48 2024-11-10T19:17:15,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/f7ff12ab6ce8476fa6367e7ae93ceed7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/f7ff12ab6ce8476fa6367e7ae93ceed7 2024-11-10T19:17:15,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/1b2e8a413291419ba97b4250ece1db28 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/1b2e8a413291419ba97b4250ece1db28 2024-11-10T19:17:15,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/20972292b673487fbf30c9caada0717b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/20972292b673487fbf30c9caada0717b 2024-11-10T19:17:15,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/b0578383a00e42dc8783c52b97c8ddf0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/b0578383a00e42dc8783c52b97c8ddf0 2024-11-10T19:17:15,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/3044ada355754b6fb9abbbf0d4e130d8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/3044ada355754b6fb9abbbf0d4e130d8 2024-11-10T19:17:15,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7e09dd8784344282861ab051c74cd0c0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/7e09dd8784344282861ab051c74cd0c0 2024-11-10T19:17:15,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/da39d722914c42c49be8b2a857389e21 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/da39d722914c42c49be8b2a857389e21 2024-11-10T19:17:15,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a403b1fb5f947588e3927fd0f97fd5a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/8a403b1fb5f947588e3927fd0f97fd5a 2024-11-10T19:17:15,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/482c5abeea804f69a8fd6729a949b030 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/482c5abeea804f69a8fd6729a949b030 2024-11-10T19:17:15,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/853da594d6234517858ffb5379bda69a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/853da594d6234517858ffb5379bda69a 2024-11-10T19:17:15,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/2fe95b85db4e478188e4398bf73e58d4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/2fe95b85db4e478188e4398bf73e58d4 2024-11-10T19:17:15,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/99ea803d4dd74a4ba7862e9fd47f9930 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/99ea803d4dd74a4ba7862e9fd47f9930 2024-11-10T19:17:15,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/21b3e3c3bf5249979f3e04ee098dc2b7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/21b3e3c3bf5249979f3e04ee098dc2b7 2024-11-10T19:17:15,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/54bf3d7847f64c9b916168171735c0fd to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/54bf3d7847f64c9b916168171735c0fd 2024-11-10T19:17:15,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/449c857f00d14a189891b800bcb4b049 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/449c857f00d14a189891b800bcb4b049 2024-11-10T19:17:15,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c2c17b1332904883b5b04bb4224a7f1d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/86c78ff610904b758bd991ba0ea84d55, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/0e76671b2c384f0588d78cd13d99df71, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f1b9f2c46d2b4212927437f000798d88, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/457adaae175c4e048a687d8014fb7351, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/7fb5a54eb14245bd9309c3c5a067f340, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/33ecdcb2e21a42eda3385aecb670b83b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/178aee35174b4726bcd9cf8a6c4b5b6a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fee0aee56fae433d9aee600958a73502, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bc30fc514d9944edbaf75f0a10a3a93f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91faf3f6c62546af9c2b0cb9c316d263, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ebf631d830c0481a9381d6aca42edb3d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e5374c291fe746ea881baf70d429f11b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e4356593b91a43f0be290c6e5cae2f55, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/6f91a02c9c1941be8fe713087900bce2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/247fd5e0be38428cbfaf909815e976c2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1b2de53ad8824925bc2f5c1003737ffd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1811c4ab001d4138aea59da7154781b7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f22411b276d0462893e6cb4d2da0e6ee, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ea03c2e063764445a6b5f1a928132778, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1dbadae4434745a8b9bd7b1cb4466fca, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c74f7a170f2c4760a962694811d5b84f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91e84283191545fa88899824d0d51bba, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/8e2aae6fa1ef45aea19b0f7ee4fa9ff2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f2c73683d0aa48e3b210c0c25f5c2407, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/25d384101d6441eb8b33d5781c082c5e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/435e8569bdd145d4940d9545552fb675, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fa14a1ac064841c3a2d51a07b73d7d42, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bbc9fc702e30405485bcabe0f1de26be, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/70097d1e4d034319ac77111b75f3d9cd] to archive 2024-11-10T19:17:15,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:17:15,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c2c17b1332904883b5b04bb4224a7f1d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c2c17b1332904883b5b04bb4224a7f1d 2024-11-10T19:17:15,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/86c78ff610904b758bd991ba0ea84d55 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/86c78ff610904b758bd991ba0ea84d55 2024-11-10T19:17:15,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/0e76671b2c384f0588d78cd13d99df71 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/0e76671b2c384f0588d78cd13d99df71 2024-11-10T19:17:15,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f1b9f2c46d2b4212927437f000798d88 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f1b9f2c46d2b4212927437f000798d88 2024-11-10T19:17:15,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/457adaae175c4e048a687d8014fb7351 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/457adaae175c4e048a687d8014fb7351 2024-11-10T19:17:15,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/7fb5a54eb14245bd9309c3c5a067f340 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/7fb5a54eb14245bd9309c3c5a067f340 2024-11-10T19:17:15,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/33ecdcb2e21a42eda3385aecb670b83b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/33ecdcb2e21a42eda3385aecb670b83b 2024-11-10T19:17:15,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/178aee35174b4726bcd9cf8a6c4b5b6a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/178aee35174b4726bcd9cf8a6c4b5b6a 2024-11-10T19:17:15,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fee0aee56fae433d9aee600958a73502 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fee0aee56fae433d9aee600958a73502 2024-11-10T19:17:15,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bc30fc514d9944edbaf75f0a10a3a93f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bc30fc514d9944edbaf75f0a10a3a93f 2024-11-10T19:17:15,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91faf3f6c62546af9c2b0cb9c316d263 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91faf3f6c62546af9c2b0cb9c316d263 2024-11-10T19:17:15,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ebf631d830c0481a9381d6aca42edb3d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ebf631d830c0481a9381d6aca42edb3d 2024-11-10T19:17:15,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e5374c291fe746ea881baf70d429f11b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e5374c291fe746ea881baf70d429f11b 2024-11-10T19:17:15,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e4356593b91a43f0be290c6e5cae2f55 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/e4356593b91a43f0be290c6e5cae2f55 2024-11-10T19:17:15,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/6f91a02c9c1941be8fe713087900bce2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/6f91a02c9c1941be8fe713087900bce2 2024-11-10T19:17:15,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/247fd5e0be38428cbfaf909815e976c2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/247fd5e0be38428cbfaf909815e976c2 2024-11-10T19:17:15,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1b2de53ad8824925bc2f5c1003737ffd to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1b2de53ad8824925bc2f5c1003737ffd 2024-11-10T19:17:15,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1811c4ab001d4138aea59da7154781b7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1811c4ab001d4138aea59da7154781b7 2024-11-10T19:17:15,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f22411b276d0462893e6cb4d2da0e6ee to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f22411b276d0462893e6cb4d2da0e6ee 2024-11-10T19:17:15,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ea03c2e063764445a6b5f1a928132778 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/ea03c2e063764445a6b5f1a928132778 2024-11-10T19:17:15,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1dbadae4434745a8b9bd7b1cb4466fca to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/1dbadae4434745a8b9bd7b1cb4466fca 2024-11-10T19:17:15,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c74f7a170f2c4760a962694811d5b84f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/c74f7a170f2c4760a962694811d5b84f 2024-11-10T19:17:15,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91e84283191545fa88899824d0d51bba to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/91e84283191545fa88899824d0d51bba 2024-11-10T19:17:15,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/8e2aae6fa1ef45aea19b0f7ee4fa9ff2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/8e2aae6fa1ef45aea19b0f7ee4fa9ff2 2024-11-10T19:17:15,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f2c73683d0aa48e3b210c0c25f5c2407 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/f2c73683d0aa48e3b210c0c25f5c2407 2024-11-10T19:17:15,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/25d384101d6441eb8b33d5781c082c5e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/25d384101d6441eb8b33d5781c082c5e 2024-11-10T19:17:15,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/435e8569bdd145d4940d9545552fb675 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/435e8569bdd145d4940d9545552fb675 2024-11-10T19:17:15,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fa14a1ac064841c3a2d51a07b73d7d42 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fa14a1ac064841c3a2d51a07b73d7d42 2024-11-10T19:17:15,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bbc9fc702e30405485bcabe0f1de26be to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/bbc9fc702e30405485bcabe0f1de26be 2024-11-10T19:17:15,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/70097d1e4d034319ac77111b75f3d9cd to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/70097d1e4d034319ac77111b75f3d9cd 2024-11-10T19:17:15,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5df3ac73de9f408f9181903e5de20f32, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/11e0f26ab7484bb594ec96bf709db49f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0310198fc3b04570815a32efa2959326, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/46ea20317a7c463492a3885be38efc44, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1780c1581eb049d9801067bcf1de9cb5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/33d04d31afe24c1f9a1cd47454a93004, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/dc7ae2f2840f482398b62429d2e1c984, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fdd2098f8570482ca881ae90d23ecdf3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/357739ee54cf489c8e1a402cab01bc25, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5aa2c8f2d8ad490dac0a7c0a897425b5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/189a76eae56e4be6a33bcb7b990efa23, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/f1ff957ff01e49ff97345b0d23432d87, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/383f5edfeea34720b65619572486a4d8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a0c701b267ba4c4395874ced17462597, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/b01592897a5e4e46a3291fb2f99f934a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1b8184a4d73d4ec092dfda62c4aa388e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/73ce0c8568674033b6051779733c9f09, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/228fe7cd284a47dbb34c459cee56f6fd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/bab71479456749d79c73b8de28e0c0b1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/9b3d09b40c754479b7a4b9eefb1beb5c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/575fef51136d4d32aabf7ab5403ef904, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0bb664f56c214553b6302c9b1959b851, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a7b931bd0b9b45c6a08e1e23f9067707, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/21a3bef38f1045378938953244eeafda, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/819833a362c646cf9a8dcdab0c1c7ac5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/04d0f5af8ff04b8daa2849b2ba1a8f1b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/7e5534f717674d84ae74a4e6c63ae6c2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/3bd300e30c30482886749a8477352d49, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fb805c1a6ab54938b2a8c892bdfc7ba2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/4898c9c9179441c3b4acd3d564f51ed0] to archive 2024-11-10T19:17:15,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:17:15,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5df3ac73de9f408f9181903e5de20f32 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5df3ac73de9f408f9181903e5de20f32 2024-11-10T19:17:15,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/11e0f26ab7484bb594ec96bf709db49f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/11e0f26ab7484bb594ec96bf709db49f 2024-11-10T19:17:15,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0310198fc3b04570815a32efa2959326 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0310198fc3b04570815a32efa2959326 2024-11-10T19:17:15,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/46ea20317a7c463492a3885be38efc44 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/46ea20317a7c463492a3885be38efc44 2024-11-10T19:17:15,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1780c1581eb049d9801067bcf1de9cb5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1780c1581eb049d9801067bcf1de9cb5 2024-11-10T19:17:15,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/33d04d31afe24c1f9a1cd47454a93004 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/33d04d31afe24c1f9a1cd47454a93004 2024-11-10T19:17:15,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/dc7ae2f2840f482398b62429d2e1c984 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/dc7ae2f2840f482398b62429d2e1c984 2024-11-10T19:17:15,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fdd2098f8570482ca881ae90d23ecdf3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fdd2098f8570482ca881ae90d23ecdf3 2024-11-10T19:17:15,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/357739ee54cf489c8e1a402cab01bc25 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/357739ee54cf489c8e1a402cab01bc25 2024-11-10T19:17:15,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5aa2c8f2d8ad490dac0a7c0a897425b5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5aa2c8f2d8ad490dac0a7c0a897425b5 2024-11-10T19:17:15,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/189a76eae56e4be6a33bcb7b990efa23 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/189a76eae56e4be6a33bcb7b990efa23 2024-11-10T19:17:15,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/f1ff957ff01e49ff97345b0d23432d87 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/f1ff957ff01e49ff97345b0d23432d87 2024-11-10T19:17:15,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/383f5edfeea34720b65619572486a4d8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/383f5edfeea34720b65619572486a4d8 2024-11-10T19:17:15,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a0c701b267ba4c4395874ced17462597 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a0c701b267ba4c4395874ced17462597 2024-11-10T19:17:15,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/b01592897a5e4e46a3291fb2f99f934a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/b01592897a5e4e46a3291fb2f99f934a 2024-11-10T19:17:15,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1b8184a4d73d4ec092dfda62c4aa388e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/1b8184a4d73d4ec092dfda62c4aa388e 2024-11-10T19:17:15,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/73ce0c8568674033b6051779733c9f09 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/73ce0c8568674033b6051779733c9f09 2024-11-10T19:17:15,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/228fe7cd284a47dbb34c459cee56f6fd to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/228fe7cd284a47dbb34c459cee56f6fd 2024-11-10T19:17:15,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/bab71479456749d79c73b8de28e0c0b1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/bab71479456749d79c73b8de28e0c0b1 2024-11-10T19:17:15,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/9b3d09b40c754479b7a4b9eefb1beb5c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/9b3d09b40c754479b7a4b9eefb1beb5c 2024-11-10T19:17:15,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/575fef51136d4d32aabf7ab5403ef904 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/575fef51136d4d32aabf7ab5403ef904 2024-11-10T19:17:15,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0bb664f56c214553b6302c9b1959b851 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/0bb664f56c214553b6302c9b1959b851 2024-11-10T19:17:15,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a7b931bd0b9b45c6a08e1e23f9067707 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/a7b931bd0b9b45c6a08e1e23f9067707 2024-11-10T19:17:15,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/21a3bef38f1045378938953244eeafda to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/21a3bef38f1045378938953244eeafda 2024-11-10T19:17:15,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/819833a362c646cf9a8dcdab0c1c7ac5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/819833a362c646cf9a8dcdab0c1c7ac5 2024-11-10T19:17:15,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/04d0f5af8ff04b8daa2849b2ba1a8f1b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/04d0f5af8ff04b8daa2849b2ba1a8f1b 2024-11-10T19:17:15,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/7e5534f717674d84ae74a4e6c63ae6c2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/7e5534f717674d84ae74a4e6c63ae6c2 2024-11-10T19:17:15,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/3bd300e30c30482886749a8477352d49 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/3bd300e30c30482886749a8477352d49 2024-11-10T19:17:15,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fb805c1a6ab54938b2a8c892bdfc7ba2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/fb805c1a6ab54938b2a8c892bdfc7ba2 2024-11-10T19:17:15,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/4898c9c9179441c3b4acd3d564f51ed0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/4898c9c9179441c3b4acd3d564f51ed0 2024-11-10T19:17:15,908 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/recovered.edits/431.seqid, newMaxSeqId=431, maxSeqId=4 2024-11-10T19:17:15,908 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. 2024-11-10T19:17:15,909 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for 17e63e9bb9f2380be13ccc4351250608: Waiting for close lock at 1731266234007Running coprocessor pre-close hooks at 1731266234007Disabling compacts and flushes for region at 1731266234007Disabling writes for close at 1731266234007Obtaining lock to block concurrent updates at 1731266234007Preparing flush snapshotting stores in 17e63e9bb9f2380be13ccc4351250608 at 1731266234007Finished memstore snapshotting TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608., syncing WAL and waiting on mvcc, flushsize=dataSize=41220, getHeapSize=123120, getOffHeapSize=0, getCellsCount=900 at 1731266234008 (+1 ms)Flushing stores of TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608. at 1731266234009 (+1 ms)Flushing 17e63e9bb9f2380be13ccc4351250608/A: creating writer at 1731266234009Flushing mob file 17e63e9bb9f2380be13ccc4351250608/A: appending metadata at 1731266234016 (+7 ms)Flushing mob file 17e63e9bb9f2380be13ccc4351250608/A: closing flushed file at 1731266234016Flushing 17e63e9bb9f2380be13ccc4351250608/A: appending metadata at 1731266234434 (+418 ms)Flushing 17e63e9bb9f2380be13ccc4351250608/A: closing flushed file at 1731266234434Flushing 17e63e9bb9f2380be13ccc4351250608/B: creating writer at 1731266234843 (+409 ms)Flushing 17e63e9bb9f2380be13ccc4351250608/B: appending metadata at 1731266234845 (+2 ms)Flushing 17e63e9bb9f2380be13ccc4351250608/B: closing flushed file at 1731266234845Flushing 17e63e9bb9f2380be13ccc4351250608/C: creating writer at 1731266235254 (+409 ms)Flushing 17e63e9bb9f2380be13ccc4351250608/C: appending metadata at 1731266235259 (+5 ms)Flushing 17e63e9bb9f2380be13ccc4351250608/C: closing flushed file at 1731266235259Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11f21c9c: reopening flushed file at 1731266235684 (+425 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e9406a5: reopening flushed file at 1731266235707 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5341f8b1: reopening flushed file at 1731266235715 (+8 ms)Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 17e63e9bb9f2380be13ccc4351250608 in 1717ms, sequenceid=428, compaction requested=false at 1731266235724 (+9 ms)Writing region close event to WAL at 1731266235904 (+180 ms)Running coprocessor post-close hooks at 1731266235908 (+4 ms)Closed at 1731266235908 2024-11-10T19:17:15,910 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed 17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:15,911 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=17e63e9bb9f2380be13ccc4351250608, regionState=CLOSED 2024-11-10T19:17:15,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:17:15,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T19:17:15,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure 17e63e9bb9f2380be13ccc4351250608, server=bdc25462c9ef,34487,1731266167729 in 2.0600 sec 2024-11-10T19:17:15,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-10T19:17:15,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17e63e9bb9f2380be13ccc4351250608, UNASSIGN in 2.0660 sec 2024-11-10T19:17:15,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T19:17:15,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0710 sec 2024-11-10T19:17:15,923 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266235923"}]},"ts":"1731266235923"} 2024-11-10T19:17:15,925 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T19:17:15,925 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T19:17:15,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0880 sec 2024-11-10T19:17:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T19:17:15,991 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:15,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T19:17:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:15,994 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T19:17:15,995 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:15,998 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,001 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/recovered.edits] 2024-11-10T19:17:16,004 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0c25063d1b7d4618a1e9df1bbc2f1edd to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/0c25063d1b7d4618a1e9df1bbc2f1edd 2024-11-10T19:17:16,006 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/206a72fdb79b48aaa77dd9f966ff85dc to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/A/206a72fdb79b48aaa77dd9f966ff85dc 2024-11-10T19:17:16,009 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/2475427f286442b59dc261a1f74237aa to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/2475427f286442b59dc261a1f74237aa 2024-11-10T19:17:16,011 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fee7b98088c349828df1a8e527679803 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/B/fee7b98088c349828df1a8e527679803 2024-11-10T19:17:16,013 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5cb71c55c27c4207a7092477c46fc846 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/5cb71c55c27c4207a7092477c46fc846 2024-11-10T19:17:16,015 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/aa444bf5620a4bada8483444f6752e7f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/C/aa444bf5620a4bada8483444f6752e7f 2024-11-10T19:17:16,020 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/recovered.edits/431.seqid to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608/recovered.edits/431.seqid 2024-11-10T19:17:16,020 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,021 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T19:17:16,021 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T19:17:16,022 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T19:17:16,026 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111009c5eb2be5944d43b165df37ac289a8a_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111009c5eb2be5944d43b165df37ac289a8a_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,028 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e1b9b35648d4d78bf4f4a22658932c2_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e1b9b35648d4d78bf4f4a22658932c2_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,032 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111033587a667fef4a36a4d55fb4e55398cd_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111033587a667fef4a36a4d55fb4e55398cd_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,034 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103539502e62654e5ba00144eb0dd0f08e_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103539502e62654e5ba00144eb0dd0f08e_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,035 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111039f0cbdf4e92495eac3865a7a3c6d783_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111039f0cbdf4e92495eac3865a7a3c6d783_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,037 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110407e9dd220964d3eb363903426838f7c_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110407e9dd220964d3eb363903426838f7c_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,039 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110473e506fc83140258fa84ffaaee5c373_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110473e506fc83140258fa84ffaaee5c373_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,042 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c6d7f20e1ac439f8ec955965512045f_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c6d7f20e1ac439f8ec955965512045f_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,044 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e80fb4d51f240669a7318b80ee70d1a_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e80fb4d51f240669a7318b80ee70d1a_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,047 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110612799affddc49efa24380749b82dee1_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110612799affddc49efa24380749b82dee1_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,049 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106ed24c01568f415bbb1529b00d4d56c8_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106ed24c01568f415bbb1529b00d4d56c8_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,050 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111085b9609aabcb42dab19a58a84517c1d4_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111085b9609aabcb42dab19a58a84517c1d4_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,052 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110997222cb53bb47cea73355e96f70057e_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110997222cb53bb47cea73355e96f70057e_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,053 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a09cdb09f5b94677b98027394507dac1_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a09cdb09f5b94677b98027394507dac1_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,054 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ab02c9798d3e402faf4a90dfd4bd2548_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ab02c9798d3e402faf4a90dfd4bd2548_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,056 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110adcffdbada7d4169a30745903ea095b2_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110adcffdbada7d4169a30745903ea095b2_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,057 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b626cd0fdd8143e48bcccbd14dbe94c2_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b626cd0fdd8143e48bcccbd14dbe94c2_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,058 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b798a983d4bf4ac3858d20a2934132d0_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b798a983d4bf4ac3858d20a2934132d0_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,060 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dad15c3950f248aa9ed13881efe249e2_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dad15c3950f248aa9ed13881efe249e2_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,061 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e32bb5d477d74b76a88385e0314981af_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e32bb5d477d74b76a88385e0314981af_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,062 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ebdadd5d819f4557835eacbb8493bc30_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ebdadd5d819f4557835eacbb8493bc30_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,063 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f14d878b48cf4fe2a9b7a9b218388056_17e63e9bb9f2380be13ccc4351250608 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f14d878b48cf4fe2a9b7a9b218388056_17e63e9bb9f2380be13ccc4351250608 2024-11-10T19:17:16,064 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T19:17:16,066 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:16,070 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T19:17:16,072 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T19:17:16,074 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:16,074 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T19:17:16,074 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731266236074"}]},"ts":"9223372036854775807"} 2024-11-10T19:17:16,077 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T19:17:16,077 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 17e63e9bb9f2380be13ccc4351250608, NAME => 'TestAcidGuarantees,,1731266202370.17e63e9bb9f2380be13ccc4351250608.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T19:17:16,077 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T19:17:16,078 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731266236077"}]},"ts":"9223372036854775807"} 2024-11-10T19:17:16,081 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T19:17:16,081 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:16,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-11-10T19:17:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T19:17:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T19:17:16,102 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:16,110 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 240) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1070050616_22 at /127.0.0.1:53918 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=797 (was 722) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1765 (was 2663) 2024-11-10T19:17:16,118 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=797, ProcessCount=11, AvailableMemoryMB=1765 2024-11-10T19:17:16,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T19:17:16,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:17:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:16,121 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T19:17:16,121 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:16,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-11-10T19:17:16,122 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T19:17:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-10T19:17:16,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742076_1252 (size=1061) 2024-11-10T19:17:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-10T19:17:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-10T19:17:16,531 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0 2024-11-10T19:17:16,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742077_1253 (size=53) 2024-11-10T19:17:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-10T19:17:16,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:17:16,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 17f3a562d0522dd06348a407d073a913, disabling compactions & flushes 2024-11-10T19:17:16,942 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:16,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:16,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. after waiting 0 ms 2024-11-10T19:17:16,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:16,942 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:16,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17f3a562d0522dd06348a407d073a913: Waiting for close lock at 1731266236942Disabling compacts and flushes for region at 1731266236942Disabling writes for close at 1731266236942Writing region close event to WAL at 1731266236942Closed at 1731266236942 2024-11-10T19:17:16,945 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T19:17:16,946 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731266236945"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731266236945"}]},"ts":"1731266236945"} 2024-11-10T19:17:16,950 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T19:17:16,952 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T19:17:16,952 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266236952"}]},"ts":"1731266236952"} 2024-11-10T19:17:16,953 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T19:17:16,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17f3a562d0522dd06348a407d073a913, ASSIGN}] 2024-11-10T19:17:16,955 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17f3a562d0522dd06348a407d073a913, ASSIGN 2024-11-10T19:17:16,956 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17f3a562d0522dd06348a407d073a913, ASSIGN; state=OFFLINE, location=bdc25462c9ef,34487,1731266167729; forceNewPlan=false, retain=false 2024-11-10T19:17:17,106 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=17f3a562d0522dd06348a407d073a913, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:17,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17f3a562d0522dd06348a407d073a913, ASSIGN because future has completed 2024-11-10T19:17:17,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:17:17,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T19:17:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-10T19:17:17,263 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:17,263 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:17:17,264 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,264 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:17:17,264 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,264 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,265 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,267 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:17:17,267 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17f3a562d0522dd06348a407d073a913 columnFamilyName A 2024-11-10T19:17:17,267 DEBUG [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:17,267 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] regionserver.HStore(327): Store=17f3a562d0522dd06348a407d073a913/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:17:17,268 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,269 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:17:17,270 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17f3a562d0522dd06348a407d073a913 columnFamilyName B 2024-11-10T19:17:17,270 DEBUG [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:17,270 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] regionserver.HStore(327): Store=17f3a562d0522dd06348a407d073a913/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:17:17,270 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,272 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:17:17,272 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17f3a562d0522dd06348a407d073a913 columnFamilyName C 2024-11-10T19:17:17,272 DEBUG [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:17,272 INFO [StoreOpener-17f3a562d0522dd06348a407d073a913-1 {}] regionserver.HStore(327): Store=17f3a562d0522dd06348a407d073a913/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:17:17,273 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:17,273 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,273 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,274 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,275 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,275 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,275 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:17:17,276 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,278 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:17:17,279 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 17f3a562d0522dd06348a407d073a913; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65318189, jitterRate=-0.026683136820793152}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:17:17,279 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:17,279 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 17f3a562d0522dd06348a407d073a913: Running coprocessor pre-open hook at 1731266237264Writing region info on filesystem at 1731266237264Initializing all the Stores at 1731266237265 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266237265Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266237265Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266237265Cleaning up temporary data from old regions at 1731266237275 (+10 ms)Running coprocessor post-open hooks at 1731266237279 (+4 ms)Region opened successfully at 1731266237279 2024-11-10T19:17:17,280 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., pid=55, masterSystemTime=1731266237260 2024-11-10T19:17:17,281 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:17,281 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:17,282 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=17f3a562d0522dd06348a407d073a913, regionState=OPEN, openSeqNum=2, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:17,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:17:17,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-10T19:17:17,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure 17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 in 175 msec 2024-11-10T19:17:17,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-10T19:17:17,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17f3a562d0522dd06348a407d073a913, ASSIGN in 332 msec 2024-11-10T19:17:17,287 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T19:17:17,287 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266237287"}]},"ts":"1731266237287"} 2024-11-10T19:17:17,289 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T19:17:17,289 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T19:17:17,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-10T19:17:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-10T19:17:18,283 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:18,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,287 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@245fbde2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,290 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad523d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,291 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,294 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T19:17:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is bdc25462c9ef,46761,1731266166843 2024-11-10T19:17:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@456abfc9 2024-11-10T19:17:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T19:17:18,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T19:17:18,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,299 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5077e18f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,301 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6f6c11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,305 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bcf19f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de5d81e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,309 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b59f8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,310 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10662616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,312 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32983ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,314 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3edadf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,317 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6947926e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,319 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63153f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,321 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eae97b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,322 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b72df5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,324 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e74eb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,325 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387ccbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,327 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b84803b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,328 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db8b3ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,330 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c276b0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c5986b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:18,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:18,333 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3492aedc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:18,334 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41461840, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:18,335 DEBUG [Thread-1176 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:18,335 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,335 DEBUG [Thread-1185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,335 DEBUG [Thread-1189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,335 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,336 DEBUG [Thread-1191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,336 DEBUG [Thread-1174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,336 DEBUG [Thread-1180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-10T19:17:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T19:17:18,338 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:18,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,339 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:18,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,341 DEBUG [Thread-1187 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,341 DEBUG [Thread-1178 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:18,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:18,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:17:18,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:17:18,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/c4c3ce4725a74b569eae7ae102693cda is 50, key is test_row_0/A:col10/1731266238349/Put/seqid=0 2024-11-10T19:17:18,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266238390, max timestamp=1731266238400Num uniques 859; 2024-11-10T19:17:18,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:18,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266238390, max timestamp=1731266238400Num uniques 859; 2024-11-10T19:17:18,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266238390, max timestamp=1731266238400Num uniques 859; 2024-11-10T19:17:18,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:18,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:18,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742078_1254 (size=11997) 2024-11-10T19:17:18,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266298430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266298431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266298433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266298433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266298433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T19:17:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:18,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:18,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:18,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:18,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266298541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266298542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266298551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266298551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266298551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:18,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:18,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:18,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:18,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T19:17:18,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266298751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266298751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266298761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266298762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266298762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:18,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:18,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/c4c3ce4725a74b569eae7ae102693cda 2024-11-10T19:17:18,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/e7f6648806a04a51bf04ef670efd6bd3 is 50, key is test_row_0/B:col10/1731266238349/Put/seqid=0 2024-11-10T19:17:18,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742079_1255 (size=9657) 2024-11-10T19:17:18,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:18,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:18,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:18,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:18,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T19:17:19,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266299061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266299061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266299071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266299071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266299072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:19,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:19,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:19,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/e7f6648806a04a51bf04ef670efd6bd3 2024-11-10T19:17:19,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/126d01941a074420b23e84546d530620 is 50, key is test_row_0/C:col10/1731266238349/Put/seqid=0 2024-11-10T19:17:19,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742080_1256 (size=9657) 2024-11-10T19:17:19,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:19,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:19,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T19:17:19,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:19,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:19,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266299571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266299571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266299581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266299581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266299591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:19,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:19,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:19,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/126d01941a074420b23e84546d530620 2024-11-10T19:17:19,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/c4c3ce4725a74b569eae7ae102693cda as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4c3ce4725a74b569eae7ae102693cda 2024-11-10T19:17:19,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4c3ce4725a74b569eae7ae102693cda, entries=150, sequenceid=11, filesize=11.7 K 2024-11-10T19:17:19,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/e7f6648806a04a51bf04ef670efd6bd3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e7f6648806a04a51bf04ef670efd6bd3 2024-11-10T19:17:19,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e7f6648806a04a51bf04ef670efd6bd3, entries=100, sequenceid=11, filesize=9.4 K 2024-11-10T19:17:19,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/126d01941a074420b23e84546d530620 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/126d01941a074420b23e84546d530620 2024-11-10T19:17:19,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/126d01941a074420b23e84546d530620, entries=100, sequenceid=11, filesize=9.4 K 2024-11-10T19:17:19,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 17f3a562d0522dd06348a407d073a913 in 1408ms, sequenceid=11, compaction requested=false 2024-11-10T19:17:19,758 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T19:17:19,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:19,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T19:17:19,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:19,874 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-10T19:17:19,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:19,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:19,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:19,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:19,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:19,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:19,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/82e87b2e8a894dd9bb9d1832e009e4a3 is 50, key is test_row_0/A:col10/1731266238429/Put/seqid=0 2024-11-10T19:17:19,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742081_1257 (size=12001) 2024-11-10T19:17:20,285 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/82e87b2e8a894dd9bb9d1832e009e4a3 2024-11-10T19:17:20,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/683e329ddc3d47c3baa037d735d88c12 is 50, key is test_row_0/B:col10/1731266238429/Put/seqid=0 2024-11-10T19:17:20,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742082_1258 (size=12001) 2024-11-10T19:17:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T19:17:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:20,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:20,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266300596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266300596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266300601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266300601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,610 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 17f3a562d0522dd06348a407d073a913 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266300611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,614 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in 17f3a562d0522dd06348a407d073a913 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,698 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/683e329ddc3d47c3baa037d735d88c12 2024-11-10T19:17:20,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/692ecea16dab41e29d340248361c8b3e is 50, key is test_row_0/C:col10/1731266238429/Put/seqid=0 2024-11-10T19:17:20,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742083_1259 (size=12001) 2024-11-10T19:17:20,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266300712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266300712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266300713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266300921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266300922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266300922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,114 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/692ecea16dab41e29d340248361c8b3e 2024-11-10T19:17:21,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/82e87b2e8a894dd9bb9d1832e009e4a3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/82e87b2e8a894dd9bb9d1832e009e4a3 2024-11-10T19:17:21,122 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/82e87b2e8a894dd9bb9d1832e009e4a3, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T19:17:21,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/683e329ddc3d47c3baa037d735d88c12 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/683e329ddc3d47c3baa037d735d88c12 2024-11-10T19:17:21,127 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/683e329ddc3d47c3baa037d735d88c12, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T19:17:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/692ecea16dab41e29d340248361c8b3e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/692ecea16dab41e29d340248361c8b3e 2024-11-10T19:17:21,133 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/692ecea16dab41e29d340248361c8b3e, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T19:17:21,134 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 17f3a562d0522dd06348a407d073a913 in 1260ms, sequenceid=48, compaction requested=false 2024-11-10T19:17:21,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:21,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:21,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-10T19:17:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-10T19:17:21,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T19:17:21,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7970 sec 2024-11-10T19:17:21,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.8030 sec 2024-11-10T19:17:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:21,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:17:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:21,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/0d1ea71ef64c429ba53735828ff910b6 is 50, key is test_row_0/A:col10/1731266240594/Put/seqid=0 2024-11-10T19:17:21,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742084_1260 (size=14341) 2024-11-10T19:17:21,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/0d1ea71ef64c429ba53735828ff910b6 2024-11-10T19:17:21,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/e1520318e2694278940321843b0d6569 is 50, key is test_row_0/B:col10/1731266240594/Put/seqid=0 2024-11-10T19:17:21,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742085_1261 (size=12001) 2024-11-10T19:17:21,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266241257, max timestamp=1731266241266Num uniques 859; 2024-11-10T19:17:21,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:21,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266241257, max timestamp=1731266241266Num uniques 859; 2024-11-10T19:17:21,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:21,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266241257, max timestamp=1731266241266Num uniques 859; 2024-11-10T19:17:21,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:21,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266301298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266301300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266301304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266301411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266301411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266301411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,435 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T19:17:21,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266301621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266301621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266301621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/e1520318e2694278940321843b0d6569 2024-11-10T19:17:21,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/709d511da21a4ccaaead21864d955d9d is 50, key is test_row_0/C:col10/1731266240594/Put/seqid=0 2024-11-10T19:17:21,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742086_1262 (size=12001) 2024-11-10T19:17:21,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266301931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266301931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266301931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/709d511da21a4ccaaead21864d955d9d 2024-11-10T19:17:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/0d1ea71ef64c429ba53735828ff910b6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/0d1ea71ef64c429ba53735828ff910b6 2024-11-10T19:17:22,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/0d1ea71ef64c429ba53735828ff910b6, entries=200, sequenceid=59, filesize=14.0 K 2024-11-10T19:17:22,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/e1520318e2694278940321843b0d6569 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1520318e2694278940321843b0d6569 2024-11-10T19:17:22,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1520318e2694278940321843b0d6569, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T19:17:22,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/709d511da21a4ccaaead21864d955d9d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/709d511da21a4ccaaead21864d955d9d 2024-11-10T19:17:22,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/709d511da21a4ccaaead21864d955d9d, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T19:17:22,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 17f3a562d0522dd06348a407d073a913 in 855ms, sequenceid=59, compaction requested=true 2024-11-10T19:17:22,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:22,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:22,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:22,089 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:22,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:22,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:22,089 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:22,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:22,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:22,090 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:22,090 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:22,090 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/A is initiating minor compaction (all files) 2024-11-10T19:17:22,090 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/B is initiating minor compaction (all files) 2024-11-10T19:17:22,090 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/A in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:22,090 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/B in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:22,090 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e7f6648806a04a51bf04ef670efd6bd3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/683e329ddc3d47c3baa037d735d88c12, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1520318e2694278940321843b0d6569] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=32.9 K 2024-11-10T19:17:22,090 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4c3ce4725a74b569eae7ae102693cda, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/82e87b2e8a894dd9bb9d1832e009e4a3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/0d1ea71ef64c429ba53735828ff910b6] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=37.4 K 2024-11-10T19:17:22,091 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting e7f6648806a04a51bf04ef670efd6bd3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731266238342 2024-11-10T19:17:22,092 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4c3ce4725a74b569eae7ae102693cda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731266238342 2024-11-10T19:17:22,092 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 683e329ddc3d47c3baa037d735d88c12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731266238429 2024-11-10T19:17:22,093 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e87b2e8a894dd9bb9d1832e009e4a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731266238429 2024-11-10T19:17:22,104 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d1ea71ef64c429ba53735828ff910b6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731266240594 2024-11-10T19:17:22,104 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting e1520318e2694278940321843b0d6569, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731266240594 2024-11-10T19:17:22,119 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:22,119 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#A#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:22,119 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/08ad76ac3ae148c89f7e1c9253a1aea1 is 50, key is test_row_0/B:col10/1731266240594/Put/seqid=0 2024-11-10T19:17:22,119 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/15843d7f87b04680aa65fe0c06587341 is 50, key is test_row_0/A:col10/1731266240594/Put/seqid=0 2024-11-10T19:17:22,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742087_1263 (size=12104) 2024-11-10T19:17:22,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742088_1264 (size=12104) 2024-11-10T19:17:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:22,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-10T19:17:22,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:22,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:22,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:22,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:22,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:22,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:22,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/9201c852aa864e4e862025b4f917945f is 50, key is test_row_0/A:col10/1731266242443/Put/seqid=0 2024-11-10T19:17:22,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742089_1265 (size=16681) 2024-11-10T19:17:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T19:17:22,506 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:22,508 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:22,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266302498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266302501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-10T19:17:22,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266302507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,513 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T19:17:22,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:22,576 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/08ad76ac3ae148c89f7e1c9253a1aea1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/08ad76ac3ae148c89f7e1c9253a1aea1 2024-11-10T19:17:22,596 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/15843d7f87b04680aa65fe0c06587341 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/15843d7f87b04680aa65fe0c06587341 2024-11-10T19:17:22,603 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/A of 17f3a562d0522dd06348a407d073a913 into 15843d7f87b04680aa65fe0c06587341(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:22,603 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:22,603 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/A, priority=13, startTime=1731266242089; duration=0sec 2024-11-10T19:17:22,603 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:22,603 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:A 2024-11-10T19:17:22,603 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:22,604 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:22,605 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/C is initiating minor compaction (all files) 2024-11-10T19:17:22,605 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/C in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:22,605 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/126d01941a074420b23e84546d530620, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/692ecea16dab41e29d340248361c8b3e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/709d511da21a4ccaaead21864d955d9d] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=32.9 K 2024-11-10T19:17:22,605 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 126d01941a074420b23e84546d530620, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731266238342 2024-11-10T19:17:22,606 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692ecea16dab41e29d340248361c8b3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731266238429 2024-11-10T19:17:22,607 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 709d511da21a4ccaaead21864d955d9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731266240594 2024-11-10T19:17:22,607 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/B of 17f3a562d0522dd06348a407d073a913 into 08ad76ac3ae148c89f7e1c9253a1aea1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:22,607 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:22,607 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/B, priority=13, startTime=1731266242089; duration=0sec 2024-11-10T19:17:22,607 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:22,607 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:B 2024-11-10T19:17:22,614 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#C#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:22,615 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/f8ad09b32cd044cc95d8dc317d383c39 is 50, key is test_row_0/C:col10/1731266240594/Put/seqid=0 2024-11-10T19:17:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T19:17:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266302622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266302622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266302622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742090_1266 (size=12104) 2024-11-10T19:17:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266302631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,644 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 17f3a562d0522dd06348a407d073a913 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266302642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,646 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in 17f3a562d0522dd06348a407d073a913 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,650 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/f8ad09b32cd044cc95d8dc317d383c39 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f8ad09b32cd044cc95d8dc317d383c39 2024-11-10T19:17:22,658 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/C of 17f3a562d0522dd06348a407d073a913 into f8ad09b32cd044cc95d8dc317d383c39(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:22,658 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:22,658 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/C, priority=13, startTime=1731266242089; duration=0sec 2024-11-10T19:17:22,658 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:22,658 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:C 2024-11-10T19:17:22,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T19:17:22,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:22,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:22,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T19:17:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:22,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T19:17:22,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266302832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266302832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266302832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:22,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/9201c852aa864e4e862025b4f917945f 2024-11-10T19:17:22,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/401e58fbccac4d72ac61ffae58e65c4a is 50, key is test_row_0/B:col10/1731266242443/Put/seqid=0 2024-11-10T19:17:22,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742091_1267 (size=12001) 2024-11-10T19:17:22,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/401e58fbccac4d72ac61ffae58e65c4a 2024-11-10T19:17:22,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/0ab39976d75442b99f1c13de4ce9ed31 is 50, key is test_row_0/C:col10/1731266242443/Put/seqid=0 2024-11-10T19:17:22,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742092_1268 (size=12001) 2024-11-10T19:17:22,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/0ab39976d75442b99f1c13de4ce9ed31 2024-11-10T19:17:22,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/9201c852aa864e4e862025b4f917945f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9201c852aa864e4e862025b4f917945f 2024-11-10T19:17:22,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9201c852aa864e4e862025b4f917945f, entries=250, sequenceid=96, filesize=16.3 K 2024-11-10T19:17:22,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/401e58fbccac4d72ac61ffae58e65c4a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/401e58fbccac4d72ac61ffae58e65c4a 2024-11-10T19:17:22,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/401e58fbccac4d72ac61ffae58e65c4a, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T19:17:22,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/0ab39976d75442b99f1c13de4ce9ed31 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0ab39976d75442b99f1c13de4ce9ed31 2024-11-10T19:17:22,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0ab39976d75442b99f1c13de4ce9ed31, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T19:17:22,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 17f3a562d0522dd06348a407d073a913 in 502ms, sequenceid=96, compaction requested=false 2024-11-10T19:17:22,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:22,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T19:17:22,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:22,979 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T19:17:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/83ee786daf9e485d89ef2f0289d6a4ae is 50, key is test_row_0/A:col10/1731266242460/Put/seqid=0 2024-11-10T19:17:22,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742093_1269 (size=12001) 2024-11-10T19:17:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T19:17:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:23,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:23,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266243160, max timestamp=1731266243172Num uniques 859; 2024-11-10T19:17:23,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:23,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266243160, max timestamp=1731266243172Num uniques 859; 2024-11-10T19:17:23,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:23,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266243160, max timestamp=1731266243172Num uniques 859; 2024-11-10T19:17:23,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:23,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266303200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266303203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266303205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266303319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266303319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266303321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,388 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/83ee786daf9e485d89ef2f0289d6a4ae 2024-11-10T19:17:23,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/96163266a59c42f9814e6d9b8afbfb9f is 50, key is test_row_0/B:col10/1731266242460/Put/seqid=0 2024-11-10T19:17:23,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742094_1270 (size=12001) 2024-11-10T19:17:23,410 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/96163266a59c42f9814e6d9b8afbfb9f 2024-11-10T19:17:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/0a0d8a95b2bb4d8d8613eb6ff7a65303 is 50, key is test_row_0/C:col10/1731266242460/Put/seqid=0 2024-11-10T19:17:23,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742095_1271 (size=12001) 2024-11-10T19:17:23,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266303532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266303532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266303532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T19:17:23,837 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/0a0d8a95b2bb4d8d8613eb6ff7a65303 2024-11-10T19:17:23,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266303841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266303841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266303842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:23,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/83ee786daf9e485d89ef2f0289d6a4ae as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/83ee786daf9e485d89ef2f0289d6a4ae 2024-11-10T19:17:23,892 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/83ee786daf9e485d89ef2f0289d6a4ae, entries=150, sequenceid=108, filesize=11.7 K 2024-11-10T19:17:23,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/96163266a59c42f9814e6d9b8afbfb9f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/96163266a59c42f9814e6d9b8afbfb9f 2024-11-10T19:17:23,937 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/96163266a59c42f9814e6d9b8afbfb9f, entries=150, sequenceid=108, filesize=11.7 K 2024-11-10T19:17:23,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/0a0d8a95b2bb4d8d8613eb6ff7a65303 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0a0d8a95b2bb4d8d8613eb6ff7a65303 2024-11-10T19:17:23,947 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0a0d8a95b2bb4d8d8613eb6ff7a65303, entries=150, sequenceid=108, filesize=11.7 K 2024-11-10T19:17:23,953 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 17f3a562d0522dd06348a407d073a913 in 973ms, sequenceid=108, compaction requested=true 2024-11-10T19:17:23,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:23,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:23,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-10T19:17:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-10T19:17:23,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-10T19:17:23,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4390 sec 2024-11-10T19:17:23,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.4490 sec 2024-11-10T19:17:24,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-10T19:17:24,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:24,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T19:17:24,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:24,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T19:17:24,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:24,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T19:17:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:24,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/9f7372b9f7514adda3879424dced4998 is 50, key is test_row_0/A:col10/1731266243203/Put/seqid=0 2024-11-10T19:17:24,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266304383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:24,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742096_1272 (size=14541) 2024-11-10T19:17:24,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266304400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:24,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266304402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:24,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266304511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:24,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266304522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266304522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T19:17:24,672 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:24,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-10T19:17:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T19:17:24,681 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:24,690 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:24,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266304724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:24,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266304732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:24,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266304732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T19:17:24,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/9f7372b9f7514adda3879424dced4998 2024-11-10T19:17:24,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-10T19:17:24,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:24,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:24,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:24,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:24,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:24,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/42be4932292b4f10a4feb929d5dbe812 is 50, key is test_row_0/B:col10/1731266243203/Put/seqid=0 2024-11-10T19:17:24,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742097_1273 (size=12151) 2024-11-10T19:17:24,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/42be4932292b4f10a4feb929d5dbe812 2024-11-10T19:17:24,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/a7aeea07fbdd457b86f6bd83ef75d486 is 50, key is test_row_0/C:col10/1731266243203/Put/seqid=0 2024-11-10T19:17:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T19:17:25,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-10T19:17:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:25,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:25,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742098_1274 (size=12151) 2024-11-10T19:17:25,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/a7aeea07fbdd457b86f6bd83ef75d486 2024-11-10T19:17:25,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266305031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:25,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266305042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266305042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:25,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/9f7372b9f7514adda3879424dced4998 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9f7372b9f7514adda3879424dced4998 2024-11-10T19:17:25,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9f7372b9f7514adda3879424dced4998, entries=200, sequenceid=146, filesize=14.2 K 2024-11-10T19:17:25,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/42be4932292b4f10a4feb929d5dbe812 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/42be4932292b4f10a4feb929d5dbe812 2024-11-10T19:17:25,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/42be4932292b4f10a4feb929d5dbe812, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T19:17:25,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/a7aeea07fbdd457b86f6bd83ef75d486 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7aeea07fbdd457b86f6bd83ef75d486 2024-11-10T19:17:25,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7aeea07fbdd457b86f6bd83ef75d486, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T19:17:25,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=40.25 KB/41220 for 17f3a562d0522dd06348a407d073a913 in 742ms, sequenceid=146, compaction requested=true 2024-11-10T19:17:25,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:25,105 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:25,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:25,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:25,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:25,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:25,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:25,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T19:17:25,108 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:25,108 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55327 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:25,108 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/A is initiating minor compaction (all files) 2024-11-10T19:17:25,108 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/A in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:25,108 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/15843d7f87b04680aa65fe0c06587341, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9201c852aa864e4e862025b4f917945f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/83ee786daf9e485d89ef2f0289d6a4ae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9f7372b9f7514adda3879424dced4998] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=54.0 K 2024-11-10T19:17:25,110 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15843d7f87b04680aa65fe0c06587341, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731266240594 2024-11-10T19:17:25,110 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9201c852aa864e4e862025b4f917945f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731266241295 2024-11-10T19:17:25,113 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83ee786daf9e485d89ef2f0289d6a4ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731266242452 2024-11-10T19:17:25,114 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:25,114 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/B is initiating minor compaction (all files) 2024-11-10T19:17:25,114 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/B in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:25,114 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/08ad76ac3ae148c89f7e1c9253a1aea1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/401e58fbccac4d72ac61ffae58e65c4a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/96163266a59c42f9814e6d9b8afbfb9f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/42be4932292b4f10a4feb929d5dbe812] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=47.1 K 2024-11-10T19:17:25,116 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f7372b9f7514adda3879424dced4998, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731266243190 2024-11-10T19:17:25,117 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ad76ac3ae148c89f7e1c9253a1aea1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731266240594 2024-11-10T19:17:25,118 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 401e58fbccac4d72ac61ffae58e65c4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731266241296 2024-11-10T19:17:25,119 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 96163266a59c42f9814e6d9b8afbfb9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731266242452 2024-11-10T19:17:25,119 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 42be4932292b4f10a4feb929d5dbe812, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731266243199 2024-11-10T19:17:25,142 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:25,142 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/22bdc248588343f9879302f4e8c69b0f is 50, key is test_row_0/A:col10/1731266243203/Put/seqid=0 2024-11-10T19:17:25,149 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:25,151 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/59d91d55739c4e8e904dc0b1b9329906 is 50, key is test_row_0/B:col10/1731266243203/Put/seqid=0 2024-11-10T19:17:25,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-10T19:17:25,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742099_1275 (size=12391) 2024-11-10T19:17:25,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:25,180 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T19:17:25,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:25,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:25,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:25,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:25,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:25,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:25,186 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/22bdc248588343f9879302f4e8c69b0f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/22bdc248588343f9879302f4e8c69b0f 2024-11-10T19:17:25,206 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17f3a562d0522dd06348a407d073a913/A of 17f3a562d0522dd06348a407d073a913 into 22bdc248588343f9879302f4e8c69b0f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:25,206 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:25,206 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/A, priority=12, startTime=1731266245105; duration=0sec 2024-11-10T19:17:25,206 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:25,206 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:A 2024-11-10T19:17:25,206 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:25,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/549c89b47f164b93b4d68fc8f05dae9a is 50, key is test_row_0/A:col10/1731266244383/Put/seqid=0 2024-11-10T19:17:25,210 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:25,210 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/C is initiating minor compaction (all files) 2024-11-10T19:17:25,210 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/C in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:25,211 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f8ad09b32cd044cc95d8dc317d383c39, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0ab39976d75442b99f1c13de4ce9ed31, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0a0d8a95b2bb4d8d8613eb6ff7a65303, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7aeea07fbdd457b86f6bd83ef75d486] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=47.1 K 2024-11-10T19:17:25,211 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8ad09b32cd044cc95d8dc317d383c39, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731266240594 2024-11-10T19:17:25,211 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ab39976d75442b99f1c13de4ce9ed31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731266241296 2024-11-10T19:17:25,212 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a0d8a95b2bb4d8d8613eb6ff7a65303, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731266242452 2024-11-10T19:17:25,212 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7aeea07fbdd457b86f6bd83ef75d486, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731266243199 2024-11-10T19:17:25,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742100_1276 (size=12391) 2024-11-10T19:17:25,234 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/59d91d55739c4e8e904dc0b1b9329906 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/59d91d55739c4e8e904dc0b1b9329906 2024-11-10T19:17:25,242 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17f3a562d0522dd06348a407d073a913/B of 17f3a562d0522dd06348a407d073a913 into 59d91d55739c4e8e904dc0b1b9329906(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:25,242 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:25,242 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/B, priority=12, startTime=1731266245106; duration=0sec 2024-11-10T19:17:25,242 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:25,242 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:B 2024-11-10T19:17:25,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742101_1277 (size=12151) 2024-11-10T19:17:25,260 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#C#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:25,260 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/032f14e7c44c4b349a8443644e8c61d4 is 50, key is test_row_0/C:col10/1731266243203/Put/seqid=0 2024-11-10T19:17:25,260 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/549c89b47f164b93b4d68fc8f05dae9a 2024-11-10T19:17:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T19:17:25,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742102_1278 (size=12391) 2024-11-10T19:17:25,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/f07950a4ef66457cad9b78e28a05c097 is 50, key is test_row_0/B:col10/1731266244383/Put/seqid=0 2024-11-10T19:17:25,348 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/032f14e7c44c4b349a8443644e8c61d4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/032f14e7c44c4b349a8443644e8c61d4 2024-11-10T19:17:25,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742103_1279 (size=12151) 2024-11-10T19:17:25,380 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17f3a562d0522dd06348a407d073a913/C of 17f3a562d0522dd06348a407d073a913 into 032f14e7c44c4b349a8443644e8c61d4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:25,380 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:25,380 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/C, priority=12, startTime=1731266245106; duration=0sec 2024-11-10T19:17:25,381 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:25,381 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:C 2024-11-10T19:17:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:25,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:25,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266245610, max timestamp=1731266245633Num uniques 859; 2024-11-10T19:17:25,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:25,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266245610, max timestamp=1731266245633Num uniques 859; 2024-11-10T19:17:25,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:25,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266305675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:25,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266305677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:25,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266305677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:25,772 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/f07950a4ef66457cad9b78e28a05c097 2024-11-10T19:17:25,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266305792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:25,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266305792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:25,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266305792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/c0b5225d1a8a4ee980ef6c7eeb0b5542 is 50, key is test_row_0/C:col10/1731266244383/Put/seqid=0 2024-11-10T19:17:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T19:17:25,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742104_1280 (size=12151) 2024-11-10T19:17:25,819 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/c0b5225d1a8a4ee980ef6c7eeb0b5542 2024-11-10T19:17:25,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/549c89b47f164b93b4d68fc8f05dae9a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/549c89b47f164b93b4d68fc8f05dae9a 2024-11-10T19:17:25,830 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/549c89b47f164b93b4d68fc8f05dae9a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T19:17:25,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/f07950a4ef66457cad9b78e28a05c097 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f07950a4ef66457cad9b78e28a05c097 2024-11-10T19:17:25,838 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f07950a4ef66457cad9b78e28a05c097, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T19:17:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/c0b5225d1a8a4ee980ef6c7eeb0b5542 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c0b5225d1a8a4ee980ef6c7eeb0b5542 2024-11-10T19:17:25,848 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c0b5225d1a8a4ee980ef6c7eeb0b5542, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T19:17:25,848 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=145.36 KB/148850 for 17f3a562d0522dd06348a407d073a913 in 668ms, sequenceid=155, compaction requested=false 2024-11-10T19:17:25,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:25,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:25,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-10T19:17:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-10T19:17:25,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-10T19:17:25,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1600 sec 2024-11-10T19:17:25,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.1790 sec 2024-11-10T19:17:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:26,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=158.78 KB heapSize=463 KB 2024-11-10T19:17:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:26,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/6bec02c8aa8b46d2975d1413f710eb14 is 50, key is test_row_0/A:col10/1731266246006/Put/seqid=0 2024-11-10T19:17:26,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266306017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266306017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742105_1281 (size=14541) 2024-11-10T19:17:26,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266306020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266306131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266306131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266306141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266306341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266306341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266306351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/6bec02c8aa8b46d2975d1413f710eb14 2024-11-10T19:17:26,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/6d63a26594784d9a923de735ae449c41 is 50, key is test_row_0/B:col10/1731266246006/Put/seqid=0 2024-11-10T19:17:26,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742106_1282 (size=12151) 2024-11-10T19:17:26,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266306651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266306651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266306661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266306672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,673 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in 17f3a562d0522dd06348a407d073a913 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266306691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:26,692 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 17f3a562d0522dd06348a407d073a913 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T19:17:26,822 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:26,822 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-10T19:17:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T19:17:26,824 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:26,824 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:26,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:26,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/6d63a26594784d9a923de735ae449c41 2024-11-10T19:17:26,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/91519768791346898161b9445b1102b1 is 50, key is test_row_0/C:col10/1731266246006/Put/seqid=0 2024-11-10T19:17:26,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742107_1283 (size=12151) 2024-11-10T19:17:26,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76.04 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/91519768791346898161b9445b1102b1 2024-11-10T19:17:26,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/6bec02c8aa8b46d2975d1413f710eb14 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6bec02c8aa8b46d2975d1413f710eb14 2024-11-10T19:17:26,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6bec02c8aa8b46d2975d1413f710eb14, entries=200, sequenceid=195, filesize=14.2 K 2024-11-10T19:17:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/6d63a26594784d9a923de735ae449c41 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/6d63a26594784d9a923de735ae449c41 2024-11-10T19:17:26,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/6d63a26594784d9a923de735ae449c41, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T19:17:26,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/91519768791346898161b9445b1102b1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/91519768791346898161b9445b1102b1 2024-11-10T19:17:26,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/91519768791346898161b9445b1102b1, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T19:17:26,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~165.49 KB/169460, heapSize ~482.88 KB/494464, currentSize=13.42 KB/13740 for 17f3a562d0522dd06348a407d073a913 in 876ms, sequenceid=195, compaction requested=true 2024-11-10T19:17:26,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:26,884 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:26,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:26,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:26,884 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:26,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:26,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:26,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:26,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:26,886 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:26,886 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:26,886 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/B is initiating minor compaction (all files) 2024-11-10T19:17:26,886 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/A is initiating minor compaction (all files) 2024-11-10T19:17:26,886 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/A in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:26,886 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/B in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:26,886 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/22bdc248588343f9879302f4e8c69b0f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/549c89b47f164b93b4d68fc8f05dae9a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6bec02c8aa8b46d2975d1413f710eb14] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=38.2 K 2024-11-10T19:17:26,886 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/59d91d55739c4e8e904dc0b1b9329906, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f07950a4ef66457cad9b78e28a05c097, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/6d63a26594784d9a923de735ae449c41] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=35.8 K 2024-11-10T19:17:26,887 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22bdc248588343f9879302f4e8c69b0f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731266243199 2024-11-10T19:17:26,887 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 59d91d55739c4e8e904dc0b1b9329906, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731266243199 2024-11-10T19:17:26,887 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 549c89b47f164b93b4d68fc8f05dae9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731266244373 2024-11-10T19:17:26,887 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting f07950a4ef66457cad9b78e28a05c097, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731266244373 2024-11-10T19:17:26,887 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bec02c8aa8b46d2975d1413f710eb14, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731266245671 2024-11-10T19:17:26,887 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d63a26594784d9a923de735ae449c41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731266245671 2024-11-10T19:17:26,903 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:26,905 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/beb67a4ba89d4bcca3aeb70ab8c9c898 is 50, key is test_row_0/A:col10/1731266246006/Put/seqid=0 2024-11-10T19:17:26,915 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#B#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:26,915 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/2e637ce7aeb24f2b8a5904e23bc912b4 is 50, key is test_row_0/B:col10/1731266246006/Put/seqid=0 2024-11-10T19:17:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T19:17:26,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742108_1284 (size=12493) 2024-11-10T19:17:26,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742109_1285 (size=12493) 2024-11-10T19:17:26,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T19:17:26,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:26,976 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-10T19:17:26,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:26,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:26,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:26,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:26,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:26,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:26,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/e2a612fc88654cf8a826729607f6b46c is 50, key is test_row_1/A:col10/1731266246017/Put/seqid=0 2024-11-10T19:17:26,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742110_1286 (size=9757) 2024-11-10T19:17:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T19:17:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:27,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:27,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266247202, max timestamp=1731266247227Num uniques 859; 2024-11-10T19:17:27,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:27,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266247202, max timestamp=1731266247227Num uniques 859; 2024-11-10T19:17:27,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:27,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266247202, max timestamp=1731266247227Num uniques 859; 2024-11-10T19:17:27,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:27,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T19:17:27,253 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T19:17:27,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731266247250, max timestamp=1731266247267Num uniques 1068; 2024-11-10T19:17:27,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-10T19:17:27,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731266247250, max timestamp=1731266247267Num uniques 1068; 2024-11-10T19:17:27,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-10T19:17:27,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731266247250, max timestamp=1731266247267Num uniques 1068; 2024-11-10T19:17:27,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-10T19:17:27,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731266247300, max timestamp=1731266247310Num uniques 1127; 2024-11-10T19:17:27,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-10T19:17:27,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731266247300, max timestamp=1731266247310Num uniques 1127; 2024-11-10T19:17:27,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-10T19:17:27,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731266247300, max timestamp=1731266247310Num uniques 1127; 2024-11-10T19:17:27,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-10T19:17:27,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731266247338, max timestamp=1731266247349Num uniques 1086; 2024-11-10T19:17:27,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-10T19:17:27,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731266247338, max timestamp=1731266247349Num uniques 1136; 2024-11-10T19:17:27,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-10T19:17:27,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731266247338, max timestamp=1731266247349Num uniques 1086; 2024-11-10T19:17:27,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-10T19:17:27,370 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/beb67a4ba89d4bcca3aeb70ab8c9c898 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/beb67a4ba89d4bcca3aeb70ab8c9c898 2024-11-10T19:17:27,372 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/2e637ce7aeb24f2b8a5904e23bc912b4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2e637ce7aeb24f2b8a5904e23bc912b4 2024-11-10T19:17:27,379 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/A of 17f3a562d0522dd06348a407d073a913 into beb67a4ba89d4bcca3aeb70ab8c9c898(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:27,379 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/B of 17f3a562d0522dd06348a407d073a913 into 2e637ce7aeb24f2b8a5904e23bc912b4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:27,379 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:27,379 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:27,379 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/B, priority=13, startTime=1731266246884; duration=0sec 2024-11-10T19:17:27,379 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/A, priority=13, startTime=1731266246884; duration=0sec 2024-11-10T19:17:27,379 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:27,379 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:A 2024-11-10T19:17:27,379 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:27,379 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:B 2024-11-10T19:17:27,379 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:27,381 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:27,381 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/C is initiating minor compaction (all files) 2024-11-10T19:17:27,381 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/C in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:27,381 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/032f14e7c44c4b349a8443644e8c61d4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c0b5225d1a8a4ee980ef6c7eeb0b5542, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/91519768791346898161b9445b1102b1] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=35.8 K 2024-11-10T19:17:27,381 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 032f14e7c44c4b349a8443644e8c61d4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731266243199 2024-11-10T19:17:27,382 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0b5225d1a8a4ee980ef6c7eeb0b5542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731266244373 2024-11-10T19:17:27,382 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91519768791346898161b9445b1102b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731266245671 2024-11-10T19:17:27,389 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:27,390 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/f5a0edbd44da4f0aaa3c90e2a0bc1aa6 is 50, key is test_row_0/C:col10/1731266246006/Put/seqid=0 2024-11-10T19:17:27,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742111_1287 (size=12493) 2024-11-10T19:17:27,399 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/e2a612fc88654cf8a826729607f6b46c 2024-11-10T19:17:27,400 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/f5a0edbd44da4f0aaa3c90e2a0bc1aa6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f5a0edbd44da4f0aaa3c90e2a0bc1aa6 2024-11-10T19:17:27,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731266247383, max timestamp=1731266247399Num uniques 1095; 2024-11-10T19:17:27,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T19:17:27,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731266247383, max timestamp=1731266247399Num uniques 1095; 2024-11-10T19:17:27,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T19:17:27,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731266247383, max timestamp=1731266247399Num uniques 1095; 2024-11-10T19:17:27,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T19:17:27,409 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/C of 17f3a562d0522dd06348a407d073a913 into f5a0edbd44da4f0aaa3c90e2a0bc1aa6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:27,409 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:27,409 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/C, priority=13, startTime=1731266246885; duration=0sec 2024-11-10T19:17:27,409 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:27,409 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:C 2024-11-10T19:17:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/498b9480ea7c48c49c9d3e5c2cfbe19a is 50, key is test_row_1/B:col10/1731266246017/Put/seqid=0 2024-11-10T19:17:27,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742112_1288 (size=9757) 2024-11-10T19:17:27,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731266247431, max timestamp=1731266247440Num uniques 1104; 2024-11-10T19:17:27,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731266247431, max timestamp=1731266247440Num uniques 1104; 2024-11-10T19:17:27,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731266247431, max timestamp=1731266247440Num uniques 1104; 2024-11-10T19:17:27,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-10T19:17:27,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-10T19:17:27,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-10T19:17:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T19:17:27,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731266247469, max timestamp=1731266247503Num uniques 1063; 2024-11-10T19:17:27,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731266247469, max timestamp=1731266247503Num uniques 1063; 2024-11-10T19:17:27,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-10T19:17:27,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-10T19:17:27,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731266247469, max timestamp=1731266247503Num uniques 1063; 2024-11-10T19:17:27,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-10T19:17:27,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731266247513, max timestamp=1731266247536Num uniques 1022; 2024-11-10T19:17:27,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-10T19:17:27,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731266247513, max timestamp=1731266247536Num uniques 1022; 2024-11-10T19:17:27,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-10T19:17:27,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731266247513, max timestamp=1731266247536Num uniques 1022; 2024-11-10T19:17:27,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-10T19:17:27,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731266247571, max timestamp=1731266247585Num uniques 1081; 2024-11-10T19:17:27,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-10T19:17:27,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731266247571, max timestamp=1731266247585Num uniques 1081; 2024-11-10T19:17:27,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731266247571, max timestamp=1731266247585Num uniques 1081; 2024-11-10T19:17:27,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-10T19:17:27,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-10T19:17:27,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731266247613, max timestamp=1731266247624Num uniques 1040; 2024-11-10T19:17:27,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731266247613, max timestamp=1731266247624Num uniques 1040; 2024-11-10T19:17:27,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-10T19:17:27,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-10T19:17:27,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731266247613, max timestamp=1731266247624Num uniques 1040; 2024-11-10T19:17:27,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-10T19:17:27,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731266247645, max timestamp=1731266247669Num uniques 1099; 2024-11-10T19:17:27,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-10T19:17:27,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731266247645, max timestamp=1731266247669Num uniques 1099; 2024-11-10T19:17:27,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-10T19:17:27,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731266247645, max timestamp=1731266247669Num uniques 1099; 2024-11-10T19:17:27,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=6870, suffix heap size=142264,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=150 2024-11-10T19:17:27,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731266247699, max timestamp=1731266247716Num uniques 1108; 2024-11-10T19:17:27,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-10T19:17:27,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731266247699, max timestamp=1731266247716Num uniques 1009; 2024-11-10T19:17:27,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46206, new segment data size=9520, suffix heap size=132904,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=208 2024-11-10T19:17:27,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731266247699, max timestamp=1731266247716Num uniques 1108; 2024-11-10T19:17:27,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-10T19:17:27,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731266247739, max timestamp=1731266247753Num uniques 1067; 2024-11-10T19:17:27,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-10T19:17:27,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731266247739, max timestamp=1731266247753Num uniques 1067; 2024-11-10T19:17:27,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-10T19:17:27,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731266247739, max timestamp=1731266247753Num uniques 1067; 2024-11-10T19:17:27,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-10T19:17:27,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731266247759, max timestamp=1731266247781Num uniques 1026; 2024-11-10T19:17:27,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731266247759, max timestamp=1731266247781Num uniques 1076; 2024-11-10T19:17:27,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-10T19:17:27,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=12636, suffix heap size=134672,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=276 2024-11-10T19:17:27,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731266247759, max timestamp=1731266247781Num uniques 1076; 2024-11-10T19:17:27,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-10T19:17:27,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731266247805, max timestamp=1731266247818Num uniques 1135; 2024-11-10T19:17:27,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=8470, suffix heap size=146008,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=185 2024-11-10T19:17:27,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731266247805, max timestamp=1731266247818Num uniques 1135; 2024-11-10T19:17:27,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=8470, suffix heap size=146008,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=185 2024-11-10T19:17:27,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8460, totalHeapSize=19480, min timestamp=1731266247805, max timestamp=1731266247818Num uniques 1135; 2024-11-10T19:17:27,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51970, new segment data size=8460, suffix heap size=146008,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=185 2024-11-10T19:17:27,826 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/498b9480ea7c48c49c9d3e5c2cfbe19a 2024-11-10T19:17:27,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/11be044295984d249557c0c049f26c6b is 50, key is test_row_1/C:col10/1731266246017/Put/seqid=0 2024-11-10T19:17:27,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742113_1289 (size=9757) 2024-11-10T19:17:27,843 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/11be044295984d249557c0c049f26c6b 2024-11-10T19:17:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/e2a612fc88654cf8a826729607f6b46c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/e2a612fc88654cf8a826729607f6b46c 2024-11-10T19:17:27,853 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/e2a612fc88654cf8a826729607f6b46c, entries=100, sequenceid=200, filesize=9.5 K 2024-11-10T19:17:27,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/498b9480ea7c48c49c9d3e5c2cfbe19a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/498b9480ea7c48c49c9d3e5c2cfbe19a 2024-11-10T19:17:27,858 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/498b9480ea7c48c49c9d3e5c2cfbe19a, entries=100, sequenceid=200, filesize=9.5 K 2024-11-10T19:17:27,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731266247832, max timestamp=1731266247857Num uniques 1044; 2024-11-10T19:17:27,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=13464, suffix heap size=136544,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=294 2024-11-10T19:17:27,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731266247832, max timestamp=1731266247857Num uniques 1044; 2024-11-10T19:17:27,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731266247832, max timestamp=1731266247857Num uniques 1044; 2024-11-10T19:17:27,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=13464, suffix heap size=136544,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=294 2024-11-10T19:17:27,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=13464, suffix heap size=136544,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=294 2024-11-10T19:17:27,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/11be044295984d249557c0c049f26c6b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/11be044295984d249557c0c049f26c6b 2024-11-10T19:17:27,866 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/11be044295984d249557c0c049f26c6b, entries=100, sequenceid=200, filesize=9.5 K 2024-11-10T19:17:27,867 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=46.96 KB/48090 for 17f3a562d0522dd06348a407d073a913 in 891ms, sequenceid=200, compaction requested=false 2024-11-10T19:17:27,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:27,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:27,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-10T19:17:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-10T19:17:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:27,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=60.38 KB heapSize=153.19 KB 2024-11-10T19:17:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:27,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-10T19:17:27,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0450 sec 2024-11-10T19:17:27,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.0490 sec 2024-11-10T19:17:27,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/bc93acacaaee46d1bd4ac8c4ff050fd2 is 50, key is test_row_0/A:col10/1731266247863/Put/seqid=0 2024-11-10T19:17:27,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742114_1290 (size=12301) 2024-11-10T19:17:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266307905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266307908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:27,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266247898, max timestamp=1731266247910Num uniques 859; 2024-11-10T19:17:27,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266247898, max timestamp=1731266247910Num uniques 859; 2024-11-10T19:17:27,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:27,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:27,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266247898, max timestamp=1731266247910Num uniques 859; 2024-11-10T19:17:27,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T19:17:27,962 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:27,962 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-10T19:17:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T19:17:27,964 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:27,965 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:27,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:27,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266307989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:27,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:27,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:27,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266308021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266308021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T19:17:28,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266308101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T19:17:28,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:28,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266308231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266308241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T19:17:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T19:17:28,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/bc93acacaaee46d1bd4ac8c4ff050fd2 2024-11-10T19:17:28,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/3ee25745bb1244dba61c2ba631ec6f5b is 50, key is test_row_0/B:col10/1731266247863/Put/seqid=0 2024-11-10T19:17:28,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742115_1291 (size=12301) 2024-11-10T19:17:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266308311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T19:17:28,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266308541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:28,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266308551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T19:17:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T19:17:28,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266308622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:28,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:28,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:28,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/3ee25745bb1244dba61c2ba631ec6f5b 2024-11-10T19:17:28,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/b922ed78c12f4c0bab74b3ab89bb81a3 is 50, key is test_row_0/C:col10/1731266247863/Put/seqid=0 2024-11-10T19:17:28,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742116_1292 (size=12301) 2024-11-10T19:17:28,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/b922ed78c12f4c0bab74b3ab89bb81a3 2024-11-10T19:17:28,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/bc93acacaaee46d1bd4ac8c4ff050fd2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/bc93acacaaee46d1bd4ac8c4ff050fd2 2024-11-10T19:17:28,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/bc93acacaaee46d1bd4ac8c4ff050fd2, entries=150, sequenceid=484, filesize=12.0 K 2024-11-10T19:17:28,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/3ee25745bb1244dba61c2ba631ec6f5b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3ee25745bb1244dba61c2ba631ec6f5b 2024-11-10T19:17:28,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3ee25745bb1244dba61c2ba631ec6f5b, entries=150, sequenceid=484, filesize=12.0 K 2024-11-10T19:17:28,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/b922ed78c12f4c0bab74b3ab89bb81a3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/b922ed78c12f4c0bab74b3ab89bb81a3 2024-11-10T19:17:28,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/b922ed78c12f4c0bab74b3ab89bb81a3, entries=150, sequenceid=484, filesize=12.0 K 2024-11-10T19:17:28,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~153.14 KB/156816, currentSize=127.47 KB/130530 for 17f3a562d0522dd06348a407d073a913 in 857ms, sequenceid=484, compaction requested=true 2024-11-10T19:17:28,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:28,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:28,726 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:28,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:28,726 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:28,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:28,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:28,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:28,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:28,727 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:28,727 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:28,727 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/B is initiating minor compaction (all files) 2024-11-10T19:17:28,727 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/A is initiating minor compaction (all files) 2024-11-10T19:17:28,727 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/B in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,727 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/A in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,727 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2e637ce7aeb24f2b8a5904e23bc912b4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/498b9480ea7c48c49c9d3e5c2cfbe19a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3ee25745bb1244dba61c2ba631ec6f5b] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=33.7 K 2024-11-10T19:17:28,727 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/beb67a4ba89d4bcca3aeb70ab8c9c898, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/e2a612fc88654cf8a826729607f6b46c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/bc93acacaaee46d1bd4ac8c4ff050fd2] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=33.7 K 2024-11-10T19:17:28,727 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e637ce7aeb24f2b8a5904e23bc912b4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731266245671 2024-11-10T19:17:28,727 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting beb67a4ba89d4bcca3aeb70ab8c9c898, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731266245671 2024-11-10T19:17:28,728 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 498b9480ea7c48c49c9d3e5c2cfbe19a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731266246012 2024-11-10T19:17:28,728 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2a612fc88654cf8a826729607f6b46c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731266246012 2024-11-10T19:17:28,728 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ee25745bb1244dba61c2ba631ec6f5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731266247857 2024-11-10T19:17:28,728 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc93acacaaee46d1bd4ac8c4ff050fd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731266247857 2024-11-10T19:17:28,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T19:17:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,730 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T19:17:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:28,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/f6eb60ac13284af499ec7ea0c0f93be0 is 50, key is test_row_0/A:col10/1731266247980/Put/seqid=0 2024-11-10T19:17:28,741 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#A#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:28,741 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/d33e61b1a36e4b5395ab828d845d27ce is 50, key is test_row_0/A:col10/1731266247863/Put/seqid=0 2024-11-10T19:17:28,750 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#B#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:28,751 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/405eca91a042496d850211a523d60bd3 is 50, key is test_row_0/B:col10/1731266247863/Put/seqid=0 2024-11-10T19:17:28,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742117_1293 (size=12301) 2024-11-10T19:17:28,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742119_1295 (size=12745) 2024-11-10T19:17:28,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742118_1294 (size=12745) 2024-11-10T19:17:28,783 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/405eca91a042496d850211a523d60bd3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/405eca91a042496d850211a523d60bd3 2024-11-10T19:17:28,790 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/B of 17f3a562d0522dd06348a407d073a913 into 405eca91a042496d850211a523d60bd3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:28,790 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:28,790 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/B, priority=13, startTime=1731266248726; duration=0sec 2024-11-10T19:17:28,790 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:28,790 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:B 2024-11-10T19:17:28,791 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:28,792 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:28,792 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/C is initiating minor compaction (all files) 2024-11-10T19:17:28,792 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/C in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:28,793 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f5a0edbd44da4f0aaa3c90e2a0bc1aa6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/11be044295984d249557c0c049f26c6b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/b922ed78c12f4c0bab74b3ab89bb81a3] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=33.7 K 2024-11-10T19:17:28,793 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a0edbd44da4f0aaa3c90e2a0bc1aa6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731266245671 2024-11-10T19:17:28,793 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 11be044295984d249557c0c049f26c6b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731266246012 2024-11-10T19:17:28,793 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting b922ed78c12f4c0bab74b3ab89bb81a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731266247857 2024-11-10T19:17:28,802 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:28,803 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/6dc259e9951e437ab668535acb27ae9b is 50, key is test_row_0/C:col10/1731266247863/Put/seqid=0 2024-11-10T19:17:28,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742120_1296 (size=12745) 2024-11-10T19:17:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:29,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:29,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266309076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:29,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266309077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:29,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T19:17:29,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266309141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:29,171 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/f6eb60ac13284af499ec7ea0c0f93be0 2024-11-10T19:17:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/d249f4d90abb4e3493f58757c18f4f43 is 50, key is test_row_0/B:col10/1731266247980/Put/seqid=0 2024-11-10T19:17:29,182 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/d33e61b1a36e4b5395ab828d845d27ce as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d33e61b1a36e4b5395ab828d845d27ce 2024-11-10T19:17:29,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742121_1297 (size=12301) 2024-11-10T19:17:29,187 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/A of 17f3a562d0522dd06348a407d073a913 into d33e61b1a36e4b5395ab828d845d27ce(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:29,187 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:29,187 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/A, priority=13, startTime=1731266248725; duration=0sec 2024-11-10T19:17:29,187 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:29,187 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:A 2024-11-10T19:17:29,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:29,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266309192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266309192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:29,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:29,228 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/6dc259e9951e437ab668535acb27ae9b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6dc259e9951e437ab668535acb27ae9b 2024-11-10T19:17:29,233 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/C of 17f3a562d0522dd06348a407d073a913 into 6dc259e9951e437ab668535acb27ae9b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:29,233 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:29,233 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/C, priority=13, startTime=1731266248726; duration=0sec 2024-11-10T19:17:29,233 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:29,233 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:C 2024-11-10T19:17:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266309401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:29,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266309401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:29,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:29,584 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/d249f4d90abb4e3493f58757c18f4f43 2024-11-10T19:17:29,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/49f0d03be04a412899bd9cd7ed951279 is 50, key is test_row_0/C:col10/1731266247980/Put/seqid=0 2024-11-10T19:17:29,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742122_1298 (size=12301) 2024-11-10T19:17:29,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266309711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266309712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,001 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/49f0d03be04a412899bd9cd7ed951279 2024-11-10T19:17:30,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/f6eb60ac13284af499ec7ea0c0f93be0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/f6eb60ac13284af499ec7ea0c0f93be0 2024-11-10T19:17:30,015 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/f6eb60ac13284af499ec7ea0c0f93be0, entries=150, sequenceid=519, filesize=12.0 K 2024-11-10T19:17:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/d249f4d90abb4e3493f58757c18f4f43 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d249f4d90abb4e3493f58757c18f4f43 2024-11-10T19:17:30,020 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d249f4d90abb4e3493f58757c18f4f43, entries=150, sequenceid=519, filesize=12.0 K 2024-11-10T19:17:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/49f0d03be04a412899bd9cd7ed951279 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/49f0d03be04a412899bd9cd7ed951279 2024-11-10T19:17:30,025 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/49f0d03be04a412899bd9cd7ed951279, entries=150, sequenceid=519, filesize=12.0 K 2024-11-10T19:17:30,026 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 17f3a562d0522dd06348a407d073a913 in 1296ms, sequenceid=519, compaction requested=false 2024-11-10T19:17:30,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:30,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-10T19:17:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-10T19:17:30,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-10T19:17:30,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0630 sec 2024-11-10T19:17:30,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.0690 sec 2024-11-10T19:17:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T19:17:30,111 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:30,112 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-10T19:17:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T19:17:30,117 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:30,117 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:30,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:30,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T19:17:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:30,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/af7c165872ad490e8325882a97ab918d is 50, key is test_row_0/A:col10/1731266249076/Put/seqid=0 2024-11-10T19:17:30,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742123_1299 (size=14741) 2024-11-10T19:17:30,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/af7c165872ad490e8325882a97ab918d 2024-11-10T19:17:30,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/8935867eec9247e787bb8b733c745637 is 50, key is test_row_0/B:col10/1731266249076/Put/seqid=0 2024-11-10T19:17:30,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742124_1300 (size=12301) 2024-11-10T19:17:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T19:17:30,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266250230, max timestamp=1731266250238Num uniques 859; 2024-11-10T19:17:30,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:30,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266250230, max timestamp=1731266250238Num uniques 859; 2024-11-10T19:17:30,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266250230, max timestamp=1731266250238Num uniques 859; 2024-11-10T19:17:30,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:30,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:30,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T19:17:30,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:30,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:30,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266310271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266310272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266310274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266310381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266310381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266310391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T19:17:30,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:30,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:30,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T19:17:30,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T19:17:30,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:30,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:30,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:30,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/8935867eec9247e787bb8b733c745637 2024-11-10T19:17:30,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/821a460829c44ee4b0370486c088bd3b is 50, key is test_row_0/C:col10/1731266249076/Put/seqid=0 2024-11-10T19:17:30,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266310592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266310592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742125_1301 (size=12301) 2024-11-10T19:17:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/821a460829c44ee4b0370486c088bd3b 2024-11-10T19:17:30,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/af7c165872ad490e8325882a97ab918d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/af7c165872ad490e8325882a97ab918d 2024-11-10T19:17:30,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/af7c165872ad490e8325882a97ab918d, entries=200, sequenceid=534, filesize=14.4 K 2024-11-10T19:17:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266310601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/8935867eec9247e787bb8b733c745637 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/8935867eec9247e787bb8b733c745637 2024-11-10T19:17:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/8935867eec9247e787bb8b733c745637, entries=150, sequenceid=534, filesize=12.0 K 2024-11-10T19:17:30,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/821a460829c44ee4b0370486c088bd3b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/821a460829c44ee4b0370486c088bd3b 2024-11-10T19:17:30,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/821a460829c44ee4b0370486c088bd3b, entries=150, sequenceid=534, filesize=12.0 K 2024-11-10T19:17:30,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 17f3a562d0522dd06348a407d073a913 in 451ms, sequenceid=534, compaction requested=true 2024-11-10T19:17:30,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:30,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:30,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:30,615 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:30,615 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:30,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:30,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:30,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:30,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:30,616 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:30,616 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:30,616 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/A is initiating minor compaction (all files) 2024-11-10T19:17:30,616 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/B is initiating minor compaction (all files) 2024-11-10T19:17:30,616 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/A in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,616 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/B in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,616 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/405eca91a042496d850211a523d60bd3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d249f4d90abb4e3493f58757c18f4f43, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/8935867eec9247e787bb8b733c745637] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=36.5 K 2024-11-10T19:17:30,616 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d33e61b1a36e4b5395ab828d845d27ce, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/f6eb60ac13284af499ec7ea0c0f93be0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/af7c165872ad490e8325882a97ab918d] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=38.9 K 2024-11-10T19:17:30,616 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 405eca91a042496d850211a523d60bd3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731266247857 2024-11-10T19:17:30,616 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting d33e61b1a36e4b5395ab828d845d27ce, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731266247857 2024-11-10T19:17:30,617 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6eb60ac13284af499ec7ea0c0f93be0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731266247972 2024-11-10T19:17:30,618 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting af7c165872ad490e8325882a97ab918d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731266249064 2024-11-10T19:17:30,618 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting d249f4d90abb4e3493f58757c18f4f43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731266247972 2024-11-10T19:17:30,618 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 8935867eec9247e787bb8b733c745637, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731266249071 2024-11-10T19:17:30,626 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:30,627 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/d9af0e7944ad41dfae4b5547144c42e1 is 50, key is test_row_0/A:col10/1731266249076/Put/seqid=0 2024-11-10T19:17:30,627 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#B#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:30,627 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/0a7aef41060d45088277d3629364975e is 50, key is test_row_0/B:col10/1731266249076/Put/seqid=0 2024-11-10T19:17:30,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742126_1302 (size=12847) 2024-11-10T19:17:30,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742127_1303 (size=12847) 2024-11-10T19:17:30,678 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/d9af0e7944ad41dfae4b5547144c42e1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d9af0e7944ad41dfae4b5547144c42e1 2024-11-10T19:17:30,682 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/A of 17f3a562d0522dd06348a407d073a913 into d9af0e7944ad41dfae4b5547144c42e1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:30,682 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:30,682 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/A, priority=13, startTime=1731266250615; duration=0sec 2024-11-10T19:17:30,682 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:30,682 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:A 2024-11-10T19:17:30,682 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:30,683 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:30,683 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/C is initiating minor compaction (all files) 2024-11-10T19:17:30,683 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/C in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,683 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6dc259e9951e437ab668535acb27ae9b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/49f0d03be04a412899bd9cd7ed951279, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/821a460829c44ee4b0370486c088bd3b] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=36.5 K 2024-11-10T19:17:30,683 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dc259e9951e437ab668535acb27ae9b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731266247857 2024-11-10T19:17:30,683 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49f0d03be04a412899bd9cd7ed951279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731266247972 2024-11-10T19:17:30,683 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 821a460829c44ee4b0370486c088bd3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731266249071 2024-11-10T19:17:30,692 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:30,692 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/ef79a98b49154a5881b701ec142e12e3 is 50, key is test_row_0/C:col10/1731266249076/Put/seqid=0 2024-11-10T19:17:30,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742128_1304 (size=12847) 2024-11-10T19:17:30,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T19:17:30,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:30,729 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-10T19:17:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T19:17:30,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/cc95f2dfeb9f43ba97e175b509e05f56 is 50, key is test_row_0/A:col10/1731266250272/Put/seqid=0 2024-11-10T19:17:30,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742129_1305 (size=12301) 2024-11-10T19:17:30,773 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/cc95f2dfeb9f43ba97e175b509e05f56 2024-11-10T19:17:30,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/a89bbb9bf4e244e6860019006500812a is 50, key is test_row_0/B:col10/1731266250272/Put/seqid=0 2024-11-10T19:17:30,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742130_1306 (size=12301) 2024-11-10T19:17:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:30,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:30,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266310923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266310924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:30,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266310928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266311041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:31,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266311041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266311041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,060 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/0a7aef41060d45088277d3629364975e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/0a7aef41060d45088277d3629364975e 2024-11-10T19:17:31,064 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/B of 17f3a562d0522dd06348a407d073a913 into 0a7aef41060d45088277d3629364975e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:31,064 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:31,064 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/B, priority=13, startTime=1731266250615; duration=0sec 2024-11-10T19:17:31,064 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:31,064 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:B 2024-11-10T19:17:31,128 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/ef79a98b49154a5881b701ec142e12e3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/ef79a98b49154a5881b701ec142e12e3 2024-11-10T19:17:31,132 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/C of 17f3a562d0522dd06348a407d073a913 into ef79a98b49154a5881b701ec142e12e3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:31,132 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:31,132 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/C, priority=13, startTime=1731266250615; duration=0sec 2024-11-10T19:17:31,132 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:31,132 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:C 2024-11-10T19:17:31,229 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/a89bbb9bf4e244e6860019006500812a 2024-11-10T19:17:31,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/c026de52783242f6be1d25efd49cf27b is 50, key is test_row_0/C:col10/1731266250272/Put/seqid=0 2024-11-10T19:17:31,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742131_1307 (size=12301) 2024-11-10T19:17:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T19:17:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266311252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266311252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266311252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266311561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266311561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266311564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:31,639 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/c026de52783242f6be1d25efd49cf27b 2024-11-10T19:17:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/cc95f2dfeb9f43ba97e175b509e05f56 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/cc95f2dfeb9f43ba97e175b509e05f56 2024-11-10T19:17:31,655 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/cc95f2dfeb9f43ba97e175b509e05f56, entries=150, sequenceid=570, filesize=12.0 K 2024-11-10T19:17:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/a89bbb9bf4e244e6860019006500812a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/a89bbb9bf4e244e6860019006500812a 2024-11-10T19:17:31,660 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/a89bbb9bf4e244e6860019006500812a, entries=150, sequenceid=570, filesize=12.0 K 2024-11-10T19:17:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/c026de52783242f6be1d25efd49cf27b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c026de52783242f6be1d25efd49cf27b 2024-11-10T19:17:31,666 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c026de52783242f6be1d25efd49cf27b, entries=150, sequenceid=570, filesize=12.0 K 2024-11-10T19:17:31,667 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 17f3a562d0522dd06348a407d073a913 in 939ms, sequenceid=570, compaction requested=false 2024-11-10T19:17:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-10T19:17:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-10T19:17:31,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T19:17:31,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5520 sec 2024-11-10T19:17:31,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5590 sec 2024-11-10T19:17:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:32,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T19:17:32,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:32,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:32,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:32,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:32,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:32,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:32,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/9cbcc3962b1549a8b112179f3b5204c5 is 50, key is test_row_0/A:col10/1731266250922/Put/seqid=0 2024-11-10T19:17:32,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742132_1308 (size=14741) 2024-11-10T19:17:32,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/9cbcc3962b1549a8b112179f3b5204c5 2024-11-10T19:17:32,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/9be89be90a48469ba3de76a9a0df4d9a is 50, key is test_row_0/B:col10/1731266250922/Put/seqid=0 2024-11-10T19:17:32,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742133_1309 (size=12301) 2024-11-10T19:17:32,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266312109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266312110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266312112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266312221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266312221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266312222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T19:17:32,261 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:32,263 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T19:17:32,265 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T19:17:32,266 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:32,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T19:17:32,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T19:17:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266312432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266312432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266312432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/9be89be90a48469ba3de76a9a0df4d9a 2024-11-10T19:17:32,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/6ac57eabd830410f892ca4843e4c7f7e is 50, key is test_row_0/C:col10/1731266250922/Put/seqid=0 2024-11-10T19:17:32,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742134_1310 (size=12301) 2024-11-10T19:17:32,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T19:17:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T19:17:32,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T19:17:32,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:32,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266312741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266312741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266312747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:32,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T19:17:32,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:32,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T19:17:32,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/6ac57eabd830410f892ca4843e4c7f7e 2024-11-10T19:17:32,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/9cbcc3962b1549a8b112179f3b5204c5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9cbcc3962b1549a8b112179f3b5204c5 2024-11-10T19:17:32,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9cbcc3962b1549a8b112179f3b5204c5, entries=200, sequenceid=585, filesize=14.4 K 2024-11-10T19:17:32,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/9be89be90a48469ba3de76a9a0df4d9a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/9be89be90a48469ba3de76a9a0df4d9a 2024-11-10T19:17:32,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/9be89be90a48469ba3de76a9a0df4d9a, entries=150, sequenceid=585, filesize=12.0 K 2024-11-10T19:17:32,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/6ac57eabd830410f892ca4843e4c7f7e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6ac57eabd830410f892ca4843e4c7f7e 2024-11-10T19:17:32,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6ac57eabd830410f892ca4843e4c7f7e, entries=150, sequenceid=585, filesize=12.0 K 2024-11-10T19:17:32,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 17f3a562d0522dd06348a407d073a913 in 865ms, sequenceid=585, compaction requested=true 2024-11-10T19:17:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:32,941 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:32,941 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:32,942 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:32,942 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/A is initiating minor compaction (all files) 2024-11-10T19:17:32,942 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/A in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,942 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d9af0e7944ad41dfae4b5547144c42e1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/cc95f2dfeb9f43ba97e175b509e05f56, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9cbcc3962b1549a8b112179f3b5204c5] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=39.0 K 2024-11-10T19:17:32,943 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:32,943 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/B is initiating minor compaction (all files) 2024-11-10T19:17:32,943 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/B in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,943 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/0a7aef41060d45088277d3629364975e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/a89bbb9bf4e244e6860019006500812a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/9be89be90a48469ba3de76a9a0df4d9a] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=36.6 K 2024-11-10T19:17:32,944 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9af0e7944ad41dfae4b5547144c42e1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731266249071 2024-11-10T19:17:32,944 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7aef41060d45088277d3629364975e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731266249071 2024-11-10T19:17:32,945 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc95f2dfeb9f43ba97e175b509e05f56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731266250257 2024-11-10T19:17:32,945 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting a89bbb9bf4e244e6860019006500812a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731266250257 2024-11-10T19:17:32,948 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be89be90a48469ba3de76a9a0df4d9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731266250922 2024-11-10T19:17:32,948 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cbcc3962b1549a8b112179f3b5204c5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731266250922 2024-11-10T19:17:32,959 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:32,959 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/f765c04ae35b4baaa8a14755bc33c589 is 50, key is test_row_0/B:col10/1731266250922/Put/seqid=0 2024-11-10T19:17:32,963 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:32,963 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/21e816cf5469499a9251bd388de61dae is 50, key is test_row_0/A:col10/1731266250922/Put/seqid=0 2024-11-10T19:17:32,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742135_1311 (size=12949) 2024-11-10T19:17:32,976 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/f765c04ae35b4baaa8a14755bc33c589 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f765c04ae35b4baaa8a14755bc33c589 2024-11-10T19:17:32,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742136_1312 (size=12949) 2024-11-10T19:17:32,981 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/B of 17f3a562d0522dd06348a407d073a913 into f765c04ae35b4baaa8a14755bc33c589(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:32,981 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:32,982 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/B, priority=13, startTime=1731266252941; duration=0sec 2024-11-10T19:17:32,982 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:32,982 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:B 2024-11-10T19:17:32,982 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:32,983 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:32,983 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/C is initiating minor compaction (all files) 2024-11-10T19:17:32,983 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/C in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:32,983 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/ef79a98b49154a5881b701ec142e12e3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c026de52783242f6be1d25efd49cf27b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6ac57eabd830410f892ca4843e4c7f7e] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=36.6 K 2024-11-10T19:17:32,983 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ef79a98b49154a5881b701ec142e12e3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731266249071 2024-11-10T19:17:32,984 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c026de52783242f6be1d25efd49cf27b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731266250257 2024-11-10T19:17:32,984 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac57eabd830410f892ca4843e4c7f7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731266250922 2024-11-10T19:17:32,994 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:32,995 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/38f02f2a393d441da31282dd6639e8f5 is 50, key is test_row_0/C:col10/1731266250922/Put/seqid=0 2024-11-10T19:17:32,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742137_1313 (size=12949) 2024-11-10T19:17:33,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T19:17:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:33,037 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T19:17:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/78f7d8e27ec04dd4914eda59583e42d8 is 50, key is test_row_0/A:col10/1731266252099/Put/seqid=0 2024-11-10T19:17:33,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742138_1314 (size=12301) 2024-11-10T19:17:33,050 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/78f7d8e27ec04dd4914eda59583e42d8 2024-11-10T19:17:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/bbd97b27e7834d88be77d0484adbe18e is 50, key is test_row_0/B:col10/1731266252099/Put/seqid=0 2024-11-10T19:17:33,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742139_1315 (size=12301) 2024-11-10T19:17:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:33,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:33,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266313271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266313274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266313276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266313381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266313382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,388 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/21e816cf5469499a9251bd388de61dae as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/21e816cf5469499a9251bd388de61dae 2024-11-10T19:17:33,394 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/A of 17f3a562d0522dd06348a407d073a913 into 21e816cf5469499a9251bd388de61dae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:33,394 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:33,395 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/A, priority=13, startTime=1731266252941; duration=0sec 2024-11-10T19:17:33,395 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:33,395 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:A 2024-11-10T19:17:33,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266313392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T19:17:33,407 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/38f02f2a393d441da31282dd6639e8f5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02f2a393d441da31282dd6639e8f5 2024-11-10T19:17:33,416 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/C of 17f3a562d0522dd06348a407d073a913 into 38f02f2a393d441da31282dd6639e8f5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:33,416 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:33,417 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/C, priority=13, startTime=1731266252941; duration=0sec 2024-11-10T19:17:33,417 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:33,417 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:C 2024-11-10T19:17:33,462 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/bbd97b27e7834d88be77d0484adbe18e 2024-11-10T19:17:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/38f02e130bc94055805a41f07348d594 is 50, key is test_row_0/C:col10/1731266252099/Put/seqid=0 2024-11-10T19:17:33,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742140_1316 (size=12301) 2024-11-10T19:17:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266313591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266313592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266313601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,872 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/38f02e130bc94055805a41f07348d594 2024-11-10T19:17:33,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/78f7d8e27ec04dd4914eda59583e42d8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/78f7d8e27ec04dd4914eda59583e42d8 2024-11-10T19:17:33,894 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/78f7d8e27ec04dd4914eda59583e42d8, entries=150, sequenceid=605, filesize=12.0 K 2024-11-10T19:17:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/bbd97b27e7834d88be77d0484adbe18e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/bbd97b27e7834d88be77d0484adbe18e 2024-11-10T19:17:33,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266313901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266313901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:33,906 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/bbd97b27e7834d88be77d0484adbe18e, entries=150, sequenceid=605, filesize=12.0 K 2024-11-10T19:17:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/38f02e130bc94055805a41f07348d594 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02e130bc94055805a41f07348d594 2024-11-10T19:17:33,912 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02e130bc94055805a41f07348d594, entries=150, sequenceid=605, filesize=12.0 K 2024-11-10T19:17:33,913 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 17f3a562d0522dd06348a407d073a913 in 876ms, sequenceid=605, compaction requested=false 2024-11-10T19:17:33,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:33,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:33,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T19:17:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T19:17:33,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T19:17:33,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6490 sec 2024-11-10T19:17:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:33,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T19:17:33,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6550 sec 2024-11-10T19:17:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:33,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:33,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/77e025f6f3b047c58885911c931ec382 is 50, key is test_row_0/A:col10/1731266253274/Put/seqid=0 2024-11-10T19:17:33,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742141_1317 (size=12301) 2024-11-10T19:17:33,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/77e025f6f3b047c58885911c931ec382 2024-11-10T19:17:33,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/e1b1510c904c47f693f904cb3a42491f is 50, key is test_row_0/B:col10/1731266253274/Put/seqid=0 2024-11-10T19:17:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742142_1318 (size=12301) 2024-11-10T19:17:33,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/e1b1510c904c47f693f904cb3a42491f 2024-11-10T19:17:33,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7db8013087f64811852901758d3d074b is 50, key is test_row_0/C:col10/1731266253274/Put/seqid=0 2024-11-10T19:17:33,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742143_1319 (size=12301) 2024-11-10T19:17:33,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7db8013087f64811852901758d3d074b 2024-11-10T19:17:33,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/77e025f6f3b047c58885911c931ec382 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/77e025f6f3b047c58885911c931ec382 2024-11-10T19:17:33,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/77e025f6f3b047c58885911c931ec382, entries=150, sequenceid=621, filesize=12.0 K 2024-11-10T19:17:34,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/e1b1510c904c47f693f904cb3a42491f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1b1510c904c47f693f904cb3a42491f 2024-11-10T19:17:34,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1b1510c904c47f693f904cb3a42491f, entries=150, sequenceid=621, filesize=12.0 K 2024-11-10T19:17:34,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7db8013087f64811852901758d3d074b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7db8013087f64811852901758d3d074b 2024-11-10T19:17:34,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266314005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:34,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7db8013087f64811852901758d3d074b, entries=150, sequenceid=621, filesize=12.0 K 2024-11-10T19:17:34,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 17f3a562d0522dd06348a407d073a913 in 96ms, sequenceid=621, compaction requested=true 2024-11-10T19:17:34,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:34,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:34,015 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:34,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:34,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:34,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:34,015 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:34,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:34,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:34,016 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:34,016 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:34,016 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/A is initiating minor compaction (all files) 2024-11-10T19:17:34,016 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/B is initiating minor compaction (all files) 2024-11-10T19:17:34,016 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/B in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:34,016 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f765c04ae35b4baaa8a14755bc33c589, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/bbd97b27e7834d88be77d0484adbe18e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1b1510c904c47f693f904cb3a42491f] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=36.7 K 2024-11-10T19:17:34,017 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/A in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:34,017 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/21e816cf5469499a9251bd388de61dae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/78f7d8e27ec04dd4914eda59583e42d8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/77e025f6f3b047c58885911c931ec382] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=36.7 K 2024-11-10T19:17:34,017 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e816cf5469499a9251bd388de61dae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731266250922 2024-11-10T19:17:34,018 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting f765c04ae35b4baaa8a14755bc33c589, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731266250922 2024-11-10T19:17:34,018 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78f7d8e27ec04dd4914eda59583e42d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731266252091 2024-11-10T19:17:34,018 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting bbd97b27e7834d88be77d0484adbe18e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731266252091 2024-11-10T19:17:34,019 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77e025f6f3b047c58885911c931ec382, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731266253269 2024-11-10T19:17:34,019 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting e1b1510c904c47f693f904cb3a42491f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731266253269 2024-11-10T19:17:34,061 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:34,061 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#A#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:34,062 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/596f37a4ff1a4c10ae99569b2594d6c2 is 50, key is test_row_0/A:col10/1731266253274/Put/seqid=0 2024-11-10T19:17:34,062 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/7243ba06d3f2422d99547e3e33585dbe is 50, key is test_row_0/B:col10/1731266253274/Put/seqid=0 2024-11-10T19:17:34,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742144_1320 (size=13051) 2024-11-10T19:17:34,079 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/7243ba06d3f2422d99547e3e33585dbe as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/7243ba06d3f2422d99547e3e33585dbe 2024-11-10T19:17:34,084 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/B of 17f3a562d0522dd06348a407d073a913 into 7243ba06d3f2422d99547e3e33585dbe(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:34,085 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:34,085 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/B, priority=13, startTime=1731266254015; duration=0sec 2024-11-10T19:17:34,085 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:34,085 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:B 2024-11-10T19:17:34,085 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:34,086 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:34,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742145_1321 (size=13051) 2024-11-10T19:17:34,086 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/C is initiating minor compaction (all files) 2024-11-10T19:17:34,087 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/C in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:34,087 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02f2a393d441da31282dd6639e8f5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02e130bc94055805a41f07348d594, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7db8013087f64811852901758d3d074b] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=36.7 K 2024-11-10T19:17:34,087 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f02f2a393d441da31282dd6639e8f5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1731266250922 2024-11-10T19:17:34,087 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f02e130bc94055805a41f07348d594, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731266252091 2024-11-10T19:17:34,088 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db8013087f64811852901758d3d074b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731266253269 2024-11-10T19:17:34,115 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#C#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:34,116 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/1d22f780587a441483d6dda7599c6c2b is 50, key is test_row_0/C:col10/1731266253274/Put/seqid=0 2024-11-10T19:17:34,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:34,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T19:17:34,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:34,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:34,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:34,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:34,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:34,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:34,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/7d7321cb56324a37bec97f42de0f2500 is 50, key is test_row_0/A:col10/1731266254003/Put/seqid=0 2024-11-10T19:17:34,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742146_1322 (size=13051) 2024-11-10T19:17:34,163 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/1d22f780587a441483d6dda7599c6c2b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/1d22f780587a441483d6dda7599c6c2b 2024-11-10T19:17:34,173 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/C of 17f3a562d0522dd06348a407d073a913 into 1d22f780587a441483d6dda7599c6c2b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:34,173 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:34,174 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/C, priority=13, startTime=1731266254015; duration=0sec 2024-11-10T19:17:34,176 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:34,177 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:C 2024-11-10T19:17:34,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742147_1323 (size=12301) 2024-11-10T19:17:34,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/7d7321cb56324a37bec97f42de0f2500 2024-11-10T19:17:34,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/33ef3f5276cd48eb9a38e86e9ebb1213 is 50, key is test_row_0/B:col10/1731266254003/Put/seqid=0 2024-11-10T19:17:34,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742148_1324 (size=12301) 2024-11-10T19:17:34,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266314199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266314311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T19:17:34,412 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:34,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T19:17:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T19:17:34,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:34,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266314412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266314412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:34,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:34,496 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/596f37a4ff1a4c10ae99569b2594d6c2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/596f37a4ff1a4c10ae99569b2594d6c2 2024-11-10T19:17:34,503 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17f3a562d0522dd06348a407d073a913/A of 17f3a562d0522dd06348a407d073a913 into 596f37a4ff1a4c10ae99569b2594d6c2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:34,503 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:34,503 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/A, priority=13, startTime=1731266254015; duration=0sec 2024-11-10T19:17:34,504 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:34,504 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:A 2024-11-10T19:17:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T19:17:34,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266314521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:34,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T19:17:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:34,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:34,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:34,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:34,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:34,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/33ef3f5276cd48eb9a38e86e9ebb1213 2024-11-10T19:17:34,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/5d1e6dcecbb546d2848c00a75e756ade is 50, key is test_row_0/C:col10/1731266254003/Put/seqid=0 2024-11-10T19:17:34,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742149_1325 (size=12301) 2024-11-10T19:17:34,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T19:17:34,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:34,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:34,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:34,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:34,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T19:17:34,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266314833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:34,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T19:17:34,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:34,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:34,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:34,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:34,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:35,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/5d1e6dcecbb546d2848c00a75e756ade 2024-11-10T19:17:35,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T19:17:35,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:35,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:35,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:35,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:35,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T19:17:35,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/7d7321cb56324a37bec97f42de0f2500 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/7d7321cb56324a37bec97f42de0f2500 2024-11-10T19:17:35,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/7d7321cb56324a37bec97f42de0f2500, entries=150, sequenceid=641, filesize=12.0 K 2024-11-10T19:17:35,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/33ef3f5276cd48eb9a38e86e9ebb1213 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/33ef3f5276cd48eb9a38e86e9ebb1213 2024-11-10T19:17:35,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/33ef3f5276cd48eb9a38e86e9ebb1213, entries=150, sequenceid=641, filesize=12.0 K 2024-11-10T19:17:35,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/5d1e6dcecbb546d2848c00a75e756ade as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5d1e6dcecbb546d2848c00a75e756ade 2024-11-10T19:17:35,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5d1e6dcecbb546d2848c00a75e756ade, entries=150, sequenceid=641, filesize=12.0 K 2024-11-10T19:17:35,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 17f3a562d0522dd06348a407d073a913 in 971ms, sequenceid=641, compaction requested=false 2024-11-10T19:17:35,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:35,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T19:17:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:35,206 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T19:17:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:35,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/20515d2a2d1f4598a7ca0f0b3350daec is 50, key is test_row_0/A:col10/1731266254197/Put/seqid=0 2024-11-10T19:17:35,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742150_1326 (size=12301) 2024-11-10T19:17:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:35,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266315513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266315517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:35,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266315521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T19:17:35,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266315622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266315624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:35,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266315631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:35,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:35,669 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=656 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/20515d2a2d1f4598a7ca0f0b3350daec 2024-11-10T19:17:35,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/2a7771d3447248bab5269faf9aecb54f is 50, key is test_row_0/B:col10/1731266254197/Put/seqid=0 2024-11-10T19:17:35,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742151_1327 (size=12301) 2024-11-10T19:17:35,787 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=656 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/2a7771d3447248bab5269faf9aecb54f 2024-11-10T19:17:35,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/548b504480a6446c9696412d6e32e030 is 50, key is test_row_0/C:col10/1731266254197/Put/seqid=0 2024-11-10T19:17:35,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742152_1328 (size=12301) 2024-11-10T19:17:35,849 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=656 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/548b504480a6446c9696412d6e32e030 2024-11-10T19:17:35,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/20515d2a2d1f4598a7ca0f0b3350daec as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/20515d2a2d1f4598a7ca0f0b3350daec 2024-11-10T19:17:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266315844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:35,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266315848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:35,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266315852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:35,860 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/20515d2a2d1f4598a7ca0f0b3350daec, entries=150, sequenceid=656, filesize=12.0 K 2024-11-10T19:17:35,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/2a7771d3447248bab5269faf9aecb54f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2a7771d3447248bab5269faf9aecb54f 2024-11-10T19:17:35,869 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2a7771d3447248bab5269faf9aecb54f, entries=150, sequenceid=656, filesize=12.0 K 2024-11-10T19:17:35,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/548b504480a6446c9696412d6e32e030 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/548b504480a6446c9696412d6e32e030 2024-11-10T19:17:35,881 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/548b504480a6446c9696412d6e32e030, entries=150, sequenceid=656, filesize=12.0 K 2024-11-10T19:17:35,890 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 17f3a562d0522dd06348a407d073a913 in 683ms, sequenceid=656, compaction requested=true 2024-11-10T19:17:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T19:17:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T19:17:35,897 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T19:17:35,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T19:17:35,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4800 sec 2024-11-10T19:17:35,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4860 sec 2024-11-10T19:17:36,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T19:17:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:36,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266255449, max timestamp=1731266256174Num uniques 859; 2024-11-10T19:17:36,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:36,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731266255449, max timestamp=1731266256174Num uniques 859; 2024-11-10T19:17:36,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T19:17:36,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731266255449, max timestamp=1731266256174Num uniques 859; 2024-11-10T19:17:36,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T19:17:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:36,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/c4682a1b8e5f489a92ba00222679cf13 is 50, key is test_row_0/A:col10/1731266255462/Put/seqid=0 2024-11-10T19:17:36,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742153_1329 (size=14741) 2024-11-10T19:17:36,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/c4682a1b8e5f489a92ba00222679cf13 2024-11-10T19:17:36,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/5a493ee8dac949b4b346aa841a8f8e07 is 50, key is test_row_0/B:col10/1731266255462/Put/seqid=0 2024-11-10T19:17:36,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266256246, max timestamp=1731266256258Num uniques 859; 2024-11-10T19:17:36,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266256246, max timestamp=1731266256258Num uniques 859; 2024-11-10T19:17:36,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:36,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266256246, max timestamp=1731266256258Num uniques 859; 2024-11-10T19:17:36,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:36,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:36,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742154_1330 (size=12301) 2024-11-10T19:17:36,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266316390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731266256296, max timestamp=1731266256390Num uniques 1018; 2024-11-10T19:17:36,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T19:17:36,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731266256310, max timestamp=1731266256390Num uniques 1018; 2024-11-10T19:17:36,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-10T19:17:36,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731266256310, max timestamp=1731266256390Num uniques 1018; 2024-11-10T19:17:36,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-10T19:17:36,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731266256462, max timestamp=1731266256518Num uniques 1077; 2024-11-10T19:17:36,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731266256462, max timestamp=1731266256518Num uniques 1009; 2024-11-10T19:17:36,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-10T19:17:36,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-10T19:17:36,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731266256462, max timestamp=1731266256518Num uniques 1009; 2024-11-10T19:17:36,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-10T19:17:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T19:17:36,576 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:36,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-10T19:17:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T19:17:36,584 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:36,585 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:36,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:36,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731266256622, max timestamp=1731266256651Num uniques 1086; 2024-11-10T19:17:36,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-10T19:17:36,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731266256622, max timestamp=1731266256651Num uniques 1086; 2024-11-10T19:17:36,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731266256622, max timestamp=1731266256651Num uniques 1086; 2024-11-10T19:17:36,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-10T19:17:36,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-10T19:17:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T19:17:36,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/5a493ee8dac949b4b346aa841a8f8e07 2024-11-10T19:17:36,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T19:17:36,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:36,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:36,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:36,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:36,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:36,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731266256735, max timestamp=1731266256752Num uniques 1095; 2024-11-10T19:17:36,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-10T19:17:36,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731266256735, max timestamp=1731266256752Num uniques 1045; 2024-11-10T19:17:36,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-10T19:17:36,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731266256735, max timestamp=1731266256752Num uniques 1045; 2024-11-10T19:17:36,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-10T19:17:36,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/5ae3e97c3e9d4878aef006b22323ef60 is 50, key is test_row_0/C:col10/1731266255462/Put/seqid=0 2024-11-10T19:17:36,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266316787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266316790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742155_1331 (size=12301) 2024-11-10T19:17:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266316793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266316795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266316797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T19:17:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:36,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:36,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266316901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T19:17:36,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266316911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266316912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266316913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266316911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T19:17:37,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:37,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:37,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:37,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:37,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:37,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266317112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266317131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266317131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266317131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266317131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/5ae3e97c3e9d4878aef006b22323ef60 2024-11-10T19:17:37,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/c4682a1b8e5f489a92ba00222679cf13 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4682a1b8e5f489a92ba00222679cf13 2024-11-10T19:17:37,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4682a1b8e5f489a92ba00222679cf13, entries=200, sequenceid=679, filesize=14.4 K 2024-11-10T19:17:37,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/5a493ee8dac949b4b346aa841a8f8e07 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/5a493ee8dac949b4b346aa841a8f8e07 2024-11-10T19:17:37,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T19:17:37,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:37,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:37,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:37,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:37,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:37,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/5a493ee8dac949b4b346aa841a8f8e07, entries=150, sequenceid=679, filesize=12.0 K 2024-11-10T19:17:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T19:17:37,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/5ae3e97c3e9d4878aef006b22323ef60 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5ae3e97c3e9d4878aef006b22323ef60 2024-11-10T19:17:37,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5ae3e97c3e9d4878aef006b22323ef60, entries=150, sequenceid=679, filesize=12.0 K 2024-11-10T19:17:37,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.46 KB/40410, heapSize ~104.20 KB/106704, currentSize=147.60 KB/151140 for 17f3a562d0522dd06348a407d073a913 in 1049ms, sequenceid=679, compaction requested=true 2024-11-10T19:17:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:37,227 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:37,227 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:37,228 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:37,228 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/A is initiating minor compaction (all files) 2024-11-10T19:17:37,228 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/A in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:37,228 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/596f37a4ff1a4c10ae99569b2594d6c2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/7d7321cb56324a37bec97f42de0f2500, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/20515d2a2d1f4598a7ca0f0b3350daec, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4682a1b8e5f489a92ba00222679cf13] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=51.2 K 2024-11-10T19:17:37,229 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:37,229 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/B is initiating minor compaction (all files) 2024-11-10T19:17:37,229 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/B in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:37,229 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/7243ba06d3f2422d99547e3e33585dbe, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/33ef3f5276cd48eb9a38e86e9ebb1213, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2a7771d3447248bab5269faf9aecb54f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/5a493ee8dac949b4b346aa841a8f8e07] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=48.8 K 2024-11-10T19:17:37,229 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 596f37a4ff1a4c10ae99569b2594d6c2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731266253269 2024-11-10T19:17:37,230 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 7243ba06d3f2422d99547e3e33585dbe, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731266253269 2024-11-10T19:17:37,230 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7321cb56324a37bec97f42de0f2500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1731266253983 2024-11-10T19:17:37,231 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ef3f5276cd48eb9a38e86e9ebb1213, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1731266253983 2024-11-10T19:17:37,231 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20515d2a2d1f4598a7ca0f0b3350daec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1731266254161 2024-11-10T19:17:37,231 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7771d3447248bab5269faf9aecb54f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1731266254161 2024-11-10T19:17:37,231 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4682a1b8e5f489a92ba00222679cf13, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=679, earliestPutTs=1731266255462 2024-11-10T19:17:37,232 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a493ee8dac949b4b346aa841a8f8e07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=679, earliestPutTs=1731266255462 2024-11-10T19:17:37,248 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:37,248 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:37,248 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/bad73bcbc18c4573a68c4e3c6ae7c5a4 is 50, key is test_row_0/B:col10/1731266255462/Put/seqid=0 2024-11-10T19:17:37,250 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/4bcd804848e740128b3c0455832bfd9e is 50, key is test_row_0/A:col10/1731266255462/Put/seqid=0 2024-11-10T19:17:37,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742156_1332 (size=13187) 2024-11-10T19:17:37,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742157_1333 (size=13187) 2024-11-10T19:17:37,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T19:17:37,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:37,370 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=147.60 KB heapSize=412.08 KB 2024-11-10T19:17:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:37,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/6f5891f23d984502b3e178ff1d8a97bb is 50, key is test_row_0/A:col10/1731266256792/Put/seqid=0 2024-11-10T19:17:37,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742158_1334 (size=12301) 2024-11-10T19:17:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:37,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. as already flushing 2024-11-10T19:17:37,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266317441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266317442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266317442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266317449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266317451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266317561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266317562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,667 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/bad73bcbc18c4573a68c4e3c6ae7c5a4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/bad73bcbc18c4573a68c4e3c6ae7c5a4 2024-11-10T19:17:37,671 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17f3a562d0522dd06348a407d073a913/B of 17f3a562d0522dd06348a407d073a913 into bad73bcbc18c4573a68c4e3c6ae7c5a4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:37,671 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:37,671 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/B, priority=12, startTime=1731266257227; duration=0sec 2024-11-10T19:17:37,671 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:37,671 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:B 2024-11-10T19:17:37,671 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:37,672 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:37,672 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 17f3a562d0522dd06348a407d073a913/C is initiating minor compaction (all files) 2024-11-10T19:17:37,673 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17f3a562d0522dd06348a407d073a913/C in TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:37,673 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/1d22f780587a441483d6dda7599c6c2b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5d1e6dcecbb546d2848c00a75e756ade, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/548b504480a6446c9696412d6e32e030, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5ae3e97c3e9d4878aef006b22323ef60] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp, totalSize=48.8 K 2024-11-10T19:17:37,673 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d22f780587a441483d6dda7599c6c2b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1731266253269 2024-11-10T19:17:37,673 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1e6dcecbb546d2848c00a75e756ade, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1731266253983 2024-11-10T19:17:37,673 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 548b504480a6446c9696412d6e32e030, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1731266254161 2024-11-10T19:17:37,674 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae3e97c3e9d4878aef006b22323ef60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=679, earliestPutTs=1731266255462 2024-11-10T19:17:37,684 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17f3a562d0522dd06348a407d073a913#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:37,684 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7eb76996fb164eae81cd5528ded19785 is 50, key is test_row_0/C:col10/1731266255462/Put/seqid=0 2024-11-10T19:17:37,684 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/4bcd804848e740128b3c0455832bfd9e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/4bcd804848e740128b3c0455832bfd9e 2024-11-10T19:17:37,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742159_1335 (size=13187) 2024-11-10T19:17:37,693 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17f3a562d0522dd06348a407d073a913/A of 17f3a562d0522dd06348a407d073a913 into 4bcd804848e740128b3c0455832bfd9e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:37,693 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:37,693 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/A, priority=12, startTime=1731266257227; duration=0sec 2024-11-10T19:17:37,693 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:37,693 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:A 2024-11-10T19:17:37,695 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7eb76996fb164eae81cd5528ded19785 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7eb76996fb164eae81cd5528ded19785 2024-11-10T19:17:37,700 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17f3a562d0522dd06348a407d073a913/C of 17f3a562d0522dd06348a407d073a913 into 7eb76996fb164eae81cd5528ded19785(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:37,700 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:37,700 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., storeName=17f3a562d0522dd06348a407d073a913/C, priority=12, startTime=1731266257227; duration=0sec 2024-11-10T19:17:37,700 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:37,700 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:C 2024-11-10T19:17:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T19:17:37,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266317771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266317771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,779 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=784 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/6f5891f23d984502b3e178ff1d8a97bb 2024-11-10T19:17:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/80fac5339d00483c9c682b09716d5d96 is 50, key is test_row_0/B:col10/1731266256792/Put/seqid=0 2024-11-10T19:17:37,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742160_1336 (size=12301) 2024-11-10T19:17:37,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37822 deadline: 1731266317961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37838 deadline: 1731266317961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:37,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37806 deadline: 1731266317961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:38,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37814 deadline: 1731266318081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:38,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37780 deadline: 1731266318081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:17:38,188 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=784 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/80fac5339d00483c9c682b09716d5d96 2024-11-10T19:17:38,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7624ddc64d89452fa3d8903f8f44832f is 50, key is test_row_0/C:col10/1731266256792/Put/seqid=0 2024-11-10T19:17:38,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742161_1337 (size=12301) 2024-11-10T19:17:38,215 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=784 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7624ddc64d89452fa3d8903f8f44832f 2024-11-10T19:17:38,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/6f5891f23d984502b3e178ff1d8a97bb as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6f5891f23d984502b3e178ff1d8a97bb 2024-11-10T19:17:38,227 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6f5891f23d984502b3e178ff1d8a97bb, entries=150, sequenceid=784, filesize=12.0 K 2024-11-10T19:17:38,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/80fac5339d00483c9c682b09716d5d96 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/80fac5339d00483c9c682b09716d5d96 2024-11-10T19:17:38,243 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/80fac5339d00483c9c682b09716d5d96, entries=150, sequenceid=784, filesize=12.0 K 2024-11-10T19:17:38,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7624ddc64d89452fa3d8903f8f44832f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7624ddc64d89452fa3d8903f8f44832f 2024-11-10T19:17:38,247 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7624ddc64d89452fa3d8903f8f44832f, entries=150, sequenceid=784, filesize=12.0 K 2024-11-10T19:17:38,249 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~412.03 KB/421920, currentSize=33.54 KB/34350 for 17f3a562d0522dd06348a407d073a913 in 878ms, sequenceid=784, compaction requested=false 2024-11-10T19:17:38,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:38,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:38,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-10T19:17:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-10T19:17:38,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T19:17:38,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6660 sec 2024-11-10T19:17:38,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6770 sec 2024-11-10T19:17:38,341 INFO [Thread-1187 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1187. 2024-11-10T19:17:38,342 DEBUG [Thread-1187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,342 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,342 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,343 INFO [Thread-1191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1191. 2024-11-10T19:17:38,343 DEBUG [Thread-1191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,343 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,343 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,344 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-11-10T19:17:38,344 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,344 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,344 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,345 INFO [Thread-1185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1185. 2024-11-10T19:17:38,345 DEBUG [Thread-1185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,345 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,345 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,346 INFO [Thread-1189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1189. 2024-11-10T19:17:38,346 DEBUG [Thread-1189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,346 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,346 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,346 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:38,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:17:38,593 INFO [Thread-1176 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1176. 2024-11-10T19:17:38,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:38,593 DEBUG [Thread-1176 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,593 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:38,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:38,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:38,594 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:38,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:38,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,594 INFO [Thread-1174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1174. 2024-11-10T19:17:38,594 DEBUG [Thread-1174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,594 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,594 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/b97a254c2bad433da918678b0b2fd0a3 is 50, key is test_row_0/A:col10/1731266257429/Put/seqid=0 2024-11-10T19:17:38,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742162_1338 (size=12301) 2024-11-10T19:17:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T19:17:38,741 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:38,972 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-10T19:17:38,973 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,973 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,973 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,983 INFO [Thread-1178 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1178. 2024-11-10T19:17:38,983 INFO [Thread-1180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1180. 2024-11-10T19:17:38,983 DEBUG [Thread-1180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,983 DEBUG [Thread-1178 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:17:38,983 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,983 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,983 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,983 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 213 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 249 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 221 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2459 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2410 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2413 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2448 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2397 2024-11-10T19:17:38,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T19:17:38,984 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:17:38,984 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:17:38,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:38,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:17:38,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T19:17:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T19:17:38,989 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266258989"}]},"ts":"1731266258989"} 2024-11-10T19:17:38,990 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T19:17:38,990 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T19:17:38,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T19:17:38,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17f3a562d0522dd06348a407d073a913, UNASSIGN}] 2024-11-10T19:17:38,994 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17f3a562d0522dd06348a407d073a913, UNASSIGN 2024-11-10T19:17:38,995 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=17f3a562d0522dd06348a407d073a913, regionState=CLOSING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:38,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17f3a562d0522dd06348a407d073a913, UNASSIGN because future has completed 2024-11-10T19:17:38,997 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T19:17:38,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:17:39,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/b97a254c2bad433da918678b0b2fd0a3 2024-11-10T19:17:39,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/d1613d8180c94d17bdc6180b85ffa175 is 50, key is test_row_0/B:col10/1731266257429/Put/seqid=0 2024-11-10T19:17:39,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742163_1339 (size=12301) 2024-11-10T19:17:39,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/d1613d8180c94d17bdc6180b85ffa175 2024-11-10T19:17:39,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7f49314bf70a4e70967a23a52be7e05a is 50, key is test_row_0/C:col10/1731266257429/Put/seqid=0 2024-11-10T19:17:39,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742164_1340 (size=12301) 2024-11-10T19:17:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T19:17:39,149 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:39,149 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T19:17:39,149 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing 17f3a562d0522dd06348a407d073a913, disabling compactions & flushes 2024-11-10T19:17:39,150 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T19:17:39,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7f49314bf70a4e70967a23a52be7e05a 2024-11-10T19:17:39,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/b97a254c2bad433da918678b0b2fd0a3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/b97a254c2bad433da918678b0b2fd0a3 2024-11-10T19:17:39,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/b97a254c2bad433da918678b0b2fd0a3, entries=150, sequenceid=797, filesize=12.0 K 2024-11-10T19:17:39,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/d1613d8180c94d17bdc6180b85ffa175 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d1613d8180c94d17bdc6180b85ffa175 2024-11-10T19:17:39,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d1613d8180c94d17bdc6180b85ffa175, entries=150, sequenceid=797, filesize=12.0 K 2024-11-10T19:17:39,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/7f49314bf70a4e70967a23a52be7e05a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7f49314bf70a4e70967a23a52be7e05a 2024-11-10T19:17:39,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7f49314bf70a4e70967a23a52be7e05a, entries=150, sequenceid=797, filesize=12.0 K 2024-11-10T19:17:39,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 17f3a562d0522dd06348a407d073a913 in 849ms, sequenceid=797, compaction requested=true 2024-11-10T19:17:39,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17f3a562d0522dd06348a407d073a913: 2024-11-10T19:17:39,442 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:39,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:39,442 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:39,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:39,442 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. after waiting 0 ms 2024-11-10T19:17:39,442 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:39,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:39,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:39,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17f3a562d0522dd06348a407d073a913:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:39,443 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. because compaction request was cancelled 2024-11-10T19:17:39,443 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. because compaction request was cancelled 2024-11-10T19:17:39,443 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing 17f3a562d0522dd06348a407d073a913 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-10T19:17:39,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:39,443 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:A 2024-11-10T19:17:39,443 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:B 2024-11-10T19:17:39,443 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. because compaction request was cancelled 2024-11-10T19:17:39,443 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17f3a562d0522dd06348a407d073a913:C 2024-11-10T19:17:39,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=A 2024-11-10T19:17:39,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:39,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=B 2024-11-10T19:17:39,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:39,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17f3a562d0522dd06348a407d073a913, store=C 2024-11-10T19:17:39,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:39,446 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/b90087a7337c48269ea1b6e1d22d775d is 50, key is test_row_1/A:col10/1731266258982/Put/seqid=0 2024-11-10T19:17:39,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742165_1341 (size=9857) 2024-11-10T19:17:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T19:17:39,849 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=803 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/b90087a7337c48269ea1b6e1d22d775d 2024-11-10T19:17:39,856 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/3d9eb456211446098bf7048ed526401e is 50, key is test_row_1/B:col10/1731266258982/Put/seqid=0 2024-11-10T19:17:39,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742166_1342 (size=9857) 2024-11-10T19:17:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T19:17:40,262 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=803 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/3d9eb456211446098bf7048ed526401e 2024-11-10T19:17:40,268 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/a7e2a36c27a04ae7bec3227eef02f901 is 50, key is test_row_1/C:col10/1731266258982/Put/seqid=0 2024-11-10T19:17:40,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742167_1343 (size=9857) 2024-11-10T19:17:40,672 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=803 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/a7e2a36c27a04ae7bec3227eef02f901 2024-11-10T19:17:40,675 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/A/b90087a7337c48269ea1b6e1d22d775d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/b90087a7337c48269ea1b6e1d22d775d 2024-11-10T19:17:40,679 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/b90087a7337c48269ea1b6e1d22d775d, entries=100, sequenceid=803, filesize=9.6 K 2024-11-10T19:17:40,679 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/B/3d9eb456211446098bf7048ed526401e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3d9eb456211446098bf7048ed526401e 2024-11-10T19:17:40,683 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3d9eb456211446098bf7048ed526401e, entries=100, sequenceid=803, filesize=9.6 K 2024-11-10T19:17:40,683 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/.tmp/C/a7e2a36c27a04ae7bec3227eef02f901 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7e2a36c27a04ae7bec3227eef02f901 2024-11-10T19:17:40,686 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7e2a36c27a04ae7bec3227eef02f901, entries=100, sequenceid=803, filesize=9.6 K 2024-11-10T19:17:40,687 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 17f3a562d0522dd06348a407d073a913 in 1245ms, sequenceid=803, compaction requested=true 2024-11-10T19:17:40,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4c3ce4725a74b569eae7ae102693cda, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/82e87b2e8a894dd9bb9d1832e009e4a3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/0d1ea71ef64c429ba53735828ff910b6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/15843d7f87b04680aa65fe0c06587341, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9201c852aa864e4e862025b4f917945f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/83ee786daf9e485d89ef2f0289d6a4ae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9f7372b9f7514adda3879424dced4998, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/22bdc248588343f9879302f4e8c69b0f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/549c89b47f164b93b4d68fc8f05dae9a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6bec02c8aa8b46d2975d1413f710eb14, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/beb67a4ba89d4bcca3aeb70ab8c9c898, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/e2a612fc88654cf8a826729607f6b46c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d33e61b1a36e4b5395ab828d845d27ce, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/bc93acacaaee46d1bd4ac8c4ff050fd2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/f6eb60ac13284af499ec7ea0c0f93be0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/af7c165872ad490e8325882a97ab918d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d9af0e7944ad41dfae4b5547144c42e1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/cc95f2dfeb9f43ba97e175b509e05f56, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9cbcc3962b1549a8b112179f3b5204c5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/21e816cf5469499a9251bd388de61dae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/78f7d8e27ec04dd4914eda59583e42d8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/596f37a4ff1a4c10ae99569b2594d6c2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/77e025f6f3b047c58885911c931ec382, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/7d7321cb56324a37bec97f42de0f2500, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/20515d2a2d1f4598a7ca0f0b3350daec, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4682a1b8e5f489a92ba00222679cf13] to archive 2024-11-10T19:17:40,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:17:40,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4c3ce4725a74b569eae7ae102693cda to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4c3ce4725a74b569eae7ae102693cda 2024-11-10T19:17:40,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/82e87b2e8a894dd9bb9d1832e009e4a3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/82e87b2e8a894dd9bb9d1832e009e4a3 2024-11-10T19:17:40,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/0d1ea71ef64c429ba53735828ff910b6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/0d1ea71ef64c429ba53735828ff910b6 2024-11-10T19:17:40,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/15843d7f87b04680aa65fe0c06587341 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/15843d7f87b04680aa65fe0c06587341 2024-11-10T19:17:40,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9201c852aa864e4e862025b4f917945f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9201c852aa864e4e862025b4f917945f 2024-11-10T19:17:40,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/83ee786daf9e485d89ef2f0289d6a4ae to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/83ee786daf9e485d89ef2f0289d6a4ae 2024-11-10T19:17:40,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9f7372b9f7514adda3879424dced4998 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9f7372b9f7514adda3879424dced4998 2024-11-10T19:17:40,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/22bdc248588343f9879302f4e8c69b0f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/22bdc248588343f9879302f4e8c69b0f 2024-11-10T19:17:40,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/549c89b47f164b93b4d68fc8f05dae9a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/549c89b47f164b93b4d68fc8f05dae9a 2024-11-10T19:17:40,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6bec02c8aa8b46d2975d1413f710eb14 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6bec02c8aa8b46d2975d1413f710eb14 2024-11-10T19:17:40,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/beb67a4ba89d4bcca3aeb70ab8c9c898 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/beb67a4ba89d4bcca3aeb70ab8c9c898 2024-11-10T19:17:40,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/e2a612fc88654cf8a826729607f6b46c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/e2a612fc88654cf8a826729607f6b46c 2024-11-10T19:17:40,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d33e61b1a36e4b5395ab828d845d27ce to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d33e61b1a36e4b5395ab828d845d27ce 2024-11-10T19:17:40,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/bc93acacaaee46d1bd4ac8c4ff050fd2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/bc93acacaaee46d1bd4ac8c4ff050fd2 2024-11-10T19:17:40,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/f6eb60ac13284af499ec7ea0c0f93be0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/f6eb60ac13284af499ec7ea0c0f93be0 2024-11-10T19:17:40,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/af7c165872ad490e8325882a97ab918d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/af7c165872ad490e8325882a97ab918d 2024-11-10T19:17:40,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d9af0e7944ad41dfae4b5547144c42e1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/d9af0e7944ad41dfae4b5547144c42e1 2024-11-10T19:17:40,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/cc95f2dfeb9f43ba97e175b509e05f56 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/cc95f2dfeb9f43ba97e175b509e05f56 2024-11-10T19:17:40,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9cbcc3962b1549a8b112179f3b5204c5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/9cbcc3962b1549a8b112179f3b5204c5 2024-11-10T19:17:40,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/21e816cf5469499a9251bd388de61dae to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/21e816cf5469499a9251bd388de61dae 2024-11-10T19:17:40,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/78f7d8e27ec04dd4914eda59583e42d8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/78f7d8e27ec04dd4914eda59583e42d8 2024-11-10T19:17:40,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/596f37a4ff1a4c10ae99569b2594d6c2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/596f37a4ff1a4c10ae99569b2594d6c2 2024-11-10T19:17:40,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/77e025f6f3b047c58885911c931ec382 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/77e025f6f3b047c58885911c931ec382 2024-11-10T19:17:40,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/7d7321cb56324a37bec97f42de0f2500 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/7d7321cb56324a37bec97f42de0f2500 2024-11-10T19:17:40,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/20515d2a2d1f4598a7ca0f0b3350daec to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/20515d2a2d1f4598a7ca0f0b3350daec 2024-11-10T19:17:40,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4682a1b8e5f489a92ba00222679cf13 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/c4682a1b8e5f489a92ba00222679cf13 2024-11-10T19:17:40,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e7f6648806a04a51bf04ef670efd6bd3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/683e329ddc3d47c3baa037d735d88c12, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/08ad76ac3ae148c89f7e1c9253a1aea1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1520318e2694278940321843b0d6569, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/401e58fbccac4d72ac61ffae58e65c4a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/96163266a59c42f9814e6d9b8afbfb9f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/59d91d55739c4e8e904dc0b1b9329906, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/42be4932292b4f10a4feb929d5dbe812, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f07950a4ef66457cad9b78e28a05c097, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2e637ce7aeb24f2b8a5904e23bc912b4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/6d63a26594784d9a923de735ae449c41, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/498b9480ea7c48c49c9d3e5c2cfbe19a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/405eca91a042496d850211a523d60bd3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3ee25745bb1244dba61c2ba631ec6f5b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d249f4d90abb4e3493f58757c18f4f43, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/0a7aef41060d45088277d3629364975e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/8935867eec9247e787bb8b733c745637, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/a89bbb9bf4e244e6860019006500812a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f765c04ae35b4baaa8a14755bc33c589, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/9be89be90a48469ba3de76a9a0df4d9a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/bbd97b27e7834d88be77d0484adbe18e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/7243ba06d3f2422d99547e3e33585dbe, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1b1510c904c47f693f904cb3a42491f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/33ef3f5276cd48eb9a38e86e9ebb1213, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2a7771d3447248bab5269faf9aecb54f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/5a493ee8dac949b4b346aa841a8f8e07] to archive 2024-11-10T19:17:40,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:17:40,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e7f6648806a04a51bf04ef670efd6bd3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e7f6648806a04a51bf04ef670efd6bd3 2024-11-10T19:17:40,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/683e329ddc3d47c3baa037d735d88c12 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/683e329ddc3d47c3baa037d735d88c12 2024-11-10T19:17:40,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/08ad76ac3ae148c89f7e1c9253a1aea1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/08ad76ac3ae148c89f7e1c9253a1aea1 2024-11-10T19:17:40,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1520318e2694278940321843b0d6569 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1520318e2694278940321843b0d6569 2024-11-10T19:17:40,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/401e58fbccac4d72ac61ffae58e65c4a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/401e58fbccac4d72ac61ffae58e65c4a 2024-11-10T19:17:40,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/96163266a59c42f9814e6d9b8afbfb9f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/96163266a59c42f9814e6d9b8afbfb9f 2024-11-10T19:17:40,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/59d91d55739c4e8e904dc0b1b9329906 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/59d91d55739c4e8e904dc0b1b9329906 2024-11-10T19:17:40,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/42be4932292b4f10a4feb929d5dbe812 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/42be4932292b4f10a4feb929d5dbe812 2024-11-10T19:17:40,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f07950a4ef66457cad9b78e28a05c097 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f07950a4ef66457cad9b78e28a05c097 2024-11-10T19:17:40,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2e637ce7aeb24f2b8a5904e23bc912b4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2e637ce7aeb24f2b8a5904e23bc912b4 2024-11-10T19:17:40,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/6d63a26594784d9a923de735ae449c41 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/6d63a26594784d9a923de735ae449c41 2024-11-10T19:17:40,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/498b9480ea7c48c49c9d3e5c2cfbe19a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/498b9480ea7c48c49c9d3e5c2cfbe19a 2024-11-10T19:17:40,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/405eca91a042496d850211a523d60bd3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/405eca91a042496d850211a523d60bd3 2024-11-10T19:17:40,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3ee25745bb1244dba61c2ba631ec6f5b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3ee25745bb1244dba61c2ba631ec6f5b 2024-11-10T19:17:40,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d249f4d90abb4e3493f58757c18f4f43 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d249f4d90abb4e3493f58757c18f4f43 2024-11-10T19:17:40,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/0a7aef41060d45088277d3629364975e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/0a7aef41060d45088277d3629364975e 2024-11-10T19:17:40,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/8935867eec9247e787bb8b733c745637 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/8935867eec9247e787bb8b733c745637 2024-11-10T19:17:40,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/a89bbb9bf4e244e6860019006500812a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/a89bbb9bf4e244e6860019006500812a 2024-11-10T19:17:40,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f765c04ae35b4baaa8a14755bc33c589 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/f765c04ae35b4baaa8a14755bc33c589 2024-11-10T19:17:40,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/9be89be90a48469ba3de76a9a0df4d9a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/9be89be90a48469ba3de76a9a0df4d9a 2024-11-10T19:17:40,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/bbd97b27e7834d88be77d0484adbe18e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/bbd97b27e7834d88be77d0484adbe18e 2024-11-10T19:17:40,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/7243ba06d3f2422d99547e3e33585dbe to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/7243ba06d3f2422d99547e3e33585dbe 2024-11-10T19:17:40,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1b1510c904c47f693f904cb3a42491f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/e1b1510c904c47f693f904cb3a42491f 2024-11-10T19:17:40,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/33ef3f5276cd48eb9a38e86e9ebb1213 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/33ef3f5276cd48eb9a38e86e9ebb1213 2024-11-10T19:17:40,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2a7771d3447248bab5269faf9aecb54f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/2a7771d3447248bab5269faf9aecb54f 2024-11-10T19:17:40,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/5a493ee8dac949b4b346aa841a8f8e07 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/5a493ee8dac949b4b346aa841a8f8e07 2024-11-10T19:17:40,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/126d01941a074420b23e84546d530620, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/692ecea16dab41e29d340248361c8b3e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f8ad09b32cd044cc95d8dc317d383c39, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/709d511da21a4ccaaead21864d955d9d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0ab39976d75442b99f1c13de4ce9ed31, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0a0d8a95b2bb4d8d8613eb6ff7a65303, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/032f14e7c44c4b349a8443644e8c61d4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7aeea07fbdd457b86f6bd83ef75d486, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c0b5225d1a8a4ee980ef6c7eeb0b5542, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f5a0edbd44da4f0aaa3c90e2a0bc1aa6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/91519768791346898161b9445b1102b1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/11be044295984d249557c0c049f26c6b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6dc259e9951e437ab668535acb27ae9b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/b922ed78c12f4c0bab74b3ab89bb81a3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/49f0d03be04a412899bd9cd7ed951279, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/ef79a98b49154a5881b701ec142e12e3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/821a460829c44ee4b0370486c088bd3b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c026de52783242f6be1d25efd49cf27b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02f2a393d441da31282dd6639e8f5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6ac57eabd830410f892ca4843e4c7f7e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02e130bc94055805a41f07348d594, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/1d22f780587a441483d6dda7599c6c2b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7db8013087f64811852901758d3d074b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5d1e6dcecbb546d2848c00a75e756ade, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/548b504480a6446c9696412d6e32e030, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5ae3e97c3e9d4878aef006b22323ef60] to archive 2024-11-10T19:17:40,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:17:40,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/126d01941a074420b23e84546d530620 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/126d01941a074420b23e84546d530620 2024-11-10T19:17:40,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/692ecea16dab41e29d340248361c8b3e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/692ecea16dab41e29d340248361c8b3e 2024-11-10T19:17:40,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f8ad09b32cd044cc95d8dc317d383c39 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f8ad09b32cd044cc95d8dc317d383c39 2024-11-10T19:17:40,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/709d511da21a4ccaaead21864d955d9d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/709d511da21a4ccaaead21864d955d9d 2024-11-10T19:17:40,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0ab39976d75442b99f1c13de4ce9ed31 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0ab39976d75442b99f1c13de4ce9ed31 2024-11-10T19:17:40,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0a0d8a95b2bb4d8d8613eb6ff7a65303 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/0a0d8a95b2bb4d8d8613eb6ff7a65303 2024-11-10T19:17:40,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/032f14e7c44c4b349a8443644e8c61d4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/032f14e7c44c4b349a8443644e8c61d4 2024-11-10T19:17:40,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7aeea07fbdd457b86f6bd83ef75d486 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7aeea07fbdd457b86f6bd83ef75d486 2024-11-10T19:17:40,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c0b5225d1a8a4ee980ef6c7eeb0b5542 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c0b5225d1a8a4ee980ef6c7eeb0b5542 2024-11-10T19:17:40,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f5a0edbd44da4f0aaa3c90e2a0bc1aa6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/f5a0edbd44da4f0aaa3c90e2a0bc1aa6 2024-11-10T19:17:40,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/91519768791346898161b9445b1102b1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/91519768791346898161b9445b1102b1 2024-11-10T19:17:40,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/11be044295984d249557c0c049f26c6b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/11be044295984d249557c0c049f26c6b 2024-11-10T19:17:40,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6dc259e9951e437ab668535acb27ae9b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6dc259e9951e437ab668535acb27ae9b 2024-11-10T19:17:40,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/b922ed78c12f4c0bab74b3ab89bb81a3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/b922ed78c12f4c0bab74b3ab89bb81a3 2024-11-10T19:17:40,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/49f0d03be04a412899bd9cd7ed951279 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/49f0d03be04a412899bd9cd7ed951279 2024-11-10T19:17:40,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/ef79a98b49154a5881b701ec142e12e3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/ef79a98b49154a5881b701ec142e12e3 2024-11-10T19:17:40,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/821a460829c44ee4b0370486c088bd3b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/821a460829c44ee4b0370486c088bd3b 2024-11-10T19:17:40,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c026de52783242f6be1d25efd49cf27b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/c026de52783242f6be1d25efd49cf27b 2024-11-10T19:17:40,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02f2a393d441da31282dd6639e8f5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02f2a393d441da31282dd6639e8f5 2024-11-10T19:17:40,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6ac57eabd830410f892ca4843e4c7f7e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/6ac57eabd830410f892ca4843e4c7f7e 2024-11-10T19:17:40,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02e130bc94055805a41f07348d594 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/38f02e130bc94055805a41f07348d594 2024-11-10T19:17:40,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/1d22f780587a441483d6dda7599c6c2b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/1d22f780587a441483d6dda7599c6c2b 2024-11-10T19:17:40,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7db8013087f64811852901758d3d074b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7db8013087f64811852901758d3d074b 2024-11-10T19:17:40,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5d1e6dcecbb546d2848c00a75e756ade to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5d1e6dcecbb546d2848c00a75e756ade 2024-11-10T19:17:40,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/548b504480a6446c9696412d6e32e030 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/548b504480a6446c9696412d6e32e030 2024-11-10T19:17:40,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5ae3e97c3e9d4878aef006b22323ef60 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/5ae3e97c3e9d4878aef006b22323ef60 2024-11-10T19:17:40,772 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/recovered.edits/806.seqid, newMaxSeqId=806, maxSeqId=1 2024-11-10T19:17:40,773 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. 2024-11-10T19:17:40,773 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for 17f3a562d0522dd06348a407d073a913: Waiting for close lock at 1731266259149Running coprocessor pre-close hooks at 1731266259149Disabling compacts and flushes for region at 1731266259149Disabling writes for close at 1731266259442 (+293 ms)Obtaining lock to block concurrent updates at 1731266259443 (+1 ms)Preparing flush snapshotting stores in 17f3a562d0522dd06348a407d073a913 at 1731266259443Finished memstore snapshotting TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731266259443Flushing stores of TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913. at 1731266259443Flushing 17f3a562d0522dd06348a407d073a913/A: creating writer at 1731266259443Flushing 17f3a562d0522dd06348a407d073a913/A: appending metadata at 1731266259445 (+2 ms)Flushing 17f3a562d0522dd06348a407d073a913/A: closing flushed file at 1731266259445Flushing 17f3a562d0522dd06348a407d073a913/B: creating writer at 1731266259852 (+407 ms)Flushing 17f3a562d0522dd06348a407d073a913/B: appending metadata at 1731266259855 (+3 ms)Flushing 17f3a562d0522dd06348a407d073a913/B: closing flushed file at 1731266259855Flushing 17f3a562d0522dd06348a407d073a913/C: creating writer at 1731266260265 (+410 ms)Flushing 17f3a562d0522dd06348a407d073a913/C: appending metadata at 1731266260268 (+3 ms)Flushing 17f3a562d0522dd06348a407d073a913/C: closing flushed file at 1731266260268Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40b9911: reopening flushed file at 1731266260675 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7aaad050: reopening flushed file at 1731266260679 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@769f0e4d: reopening flushed file at 1731266260683 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 17f3a562d0522dd06348a407d073a913 in 1245ms, sequenceid=803, compaction requested=true at 1731266260687 (+4 ms)Writing region close event to WAL at 1731266260769 (+82 ms)Running coprocessor post-close hooks at 1731266260773 (+4 ms)Closed at 1731266260773 2024-11-10T19:17:40,775 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed 17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:40,775 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=17f3a562d0522dd06348a407d073a913, regionState=CLOSED 2024-11-10T19:17:40,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:17:40,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-10T19:17:40,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure 17f3a562d0522dd06348a407d073a913, server=bdc25462c9ef,34487,1731266167729 in 1.7800 sec 2024-11-10T19:17:40,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-10T19:17:40,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17f3a562d0522dd06348a407d073a913, UNASSIGN in 1.7850 sec 2024-11-10T19:17:40,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T19:17:40,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7880 sec 2024-11-10T19:17:40,782 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266260782"}]},"ts":"1731266260782"} 2024-11-10T19:17:40,783 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T19:17:40,783 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T19:17:40,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7970 sec 2024-11-10T19:17:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T19:17:41,131 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:41,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T19:17:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:41,133 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T19:17:41,134 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:41,136 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:41,139 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/recovered.edits] 2024-11-10T19:17:41,142 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/4bcd804848e740128b3c0455832bfd9e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/4bcd804848e740128b3c0455832bfd9e 2024-11-10T19:17:41,144 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6f5891f23d984502b3e178ff1d8a97bb to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/6f5891f23d984502b3e178ff1d8a97bb 2024-11-10T19:17:41,145 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/b90087a7337c48269ea1b6e1d22d775d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/b90087a7337c48269ea1b6e1d22d775d 2024-11-10T19:17:41,147 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/b97a254c2bad433da918678b0b2fd0a3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/A/b97a254c2bad433da918678b0b2fd0a3 2024-11-10T19:17:41,157 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3d9eb456211446098bf7048ed526401e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/3d9eb456211446098bf7048ed526401e 2024-11-10T19:17:41,158 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/80fac5339d00483c9c682b09716d5d96 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/80fac5339d00483c9c682b09716d5d96 2024-11-10T19:17:41,159 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/bad73bcbc18c4573a68c4e3c6ae7c5a4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/bad73bcbc18c4573a68c4e3c6ae7c5a4 2024-11-10T19:17:41,160 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d1613d8180c94d17bdc6180b85ffa175 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/B/d1613d8180c94d17bdc6180b85ffa175 2024-11-10T19:17:41,162 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7624ddc64d89452fa3d8903f8f44832f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7624ddc64d89452fa3d8903f8f44832f 2024-11-10T19:17:41,163 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7eb76996fb164eae81cd5528ded19785 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7eb76996fb164eae81cd5528ded19785 2024-11-10T19:17:41,164 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7f49314bf70a4e70967a23a52be7e05a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/7f49314bf70a4e70967a23a52be7e05a 2024-11-10T19:17:41,165 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7e2a36c27a04ae7bec3227eef02f901 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/C/a7e2a36c27a04ae7bec3227eef02f901 2024-11-10T19:17:41,170 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/recovered.edits/806.seqid to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913/recovered.edits/806.seqid 2024-11-10T19:17:41,170 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/17f3a562d0522dd06348a407d073a913 2024-11-10T19:17:41,170 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T19:17:41,175 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:41,178 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T19:17:41,180 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T19:17:41,186 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:41,186 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T19:17:41,186 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731266261186"}]},"ts":"9223372036854775807"} 2024-11-10T19:17:41,190 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T19:17:41,190 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 17f3a562d0522dd06348a407d073a913, NAME => 'TestAcidGuarantees,,1731266236119.17f3a562d0522dd06348a407d073a913.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T19:17:41,190 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T19:17:41,191 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731266261190"}]},"ts":"9223372036854775807"} 2024-11-10T19:17:41,193 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T19:17:41,196 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:41,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-10T19:17:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T19:17:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T19:17:41,248 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:41,257 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1070050616_22 at /127.0.0.1:38450 [Waiting for operation #247] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1841089147_22 at /127.0.0.1:53918 [Waiting for operation #325] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=780 (was 797), ProcessCount=11 (was 11), AvailableMemoryMB=1184 (was 1765) 2024-11-10T19:17:41,273 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=780, ProcessCount=11, AvailableMemoryMB=1182 2024-11-10T19:17:41,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T19:17:41,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:17:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:41,279 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T19:17:41,279 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:41,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-10T19:17:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-10T19:17:41,281 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T19:17:41,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742168_1344 (size=1061) 2024-11-10T19:17:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-10T19:17:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-10T19:17:41,726 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0 2024-11-10T19:17:41,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742169_1345 (size=53) 2024-11-10T19:17:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-10T19:17:42,132 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:17:42,132 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bccf9fccd1002d9d7579e690ca8d056c, disabling compactions & flushes 2024-11-10T19:17:42,132 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:42,132 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:42,132 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. after waiting 0 ms 2024-11-10T19:17:42,132 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:42,132 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:42,132 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bccf9fccd1002d9d7579e690ca8d056c: Waiting for close lock at 1731266262132Disabling compacts and flushes for region at 1731266262132Disabling writes for close at 1731266262132Writing region close event to WAL at 1731266262132Closed at 1731266262132 2024-11-10T19:17:42,134 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T19:17:42,134 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731266262134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731266262134"}]},"ts":"1731266262134"} 2024-11-10T19:17:42,136 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T19:17:42,137 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T19:17:42,137 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266262137"}]},"ts":"1731266262137"} 2024-11-10T19:17:42,139 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T19:17:42,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, ASSIGN}] 2024-11-10T19:17:42,141 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, ASSIGN 2024-11-10T19:17:42,142 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, ASSIGN; state=OFFLINE, location=bdc25462c9ef,34487,1731266167729; forceNewPlan=false, retain=false 2024-11-10T19:17:42,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=bccf9fccd1002d9d7579e690ca8d056c, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:42,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, ASSIGN because future has completed 2024-11-10T19:17:42,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:17:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-10T19:17:42,450 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:42,451 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:17:42,451 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,451 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:17:42,451 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,451 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,452 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,454 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:17:42,454 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bccf9fccd1002d9d7579e690ca8d056c columnFamilyName A 2024-11-10T19:17:42,454 DEBUG [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:42,454 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(327): Store=bccf9fccd1002d9d7579e690ca8d056c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:17:42,454 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,456 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:17:42,456 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bccf9fccd1002d9d7579e690ca8d056c columnFamilyName B 2024-11-10T19:17:42,456 DEBUG [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:42,456 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(327): Store=bccf9fccd1002d9d7579e690ca8d056c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:17:42,456 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,457 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:17:42,458 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bccf9fccd1002d9d7579e690ca8d056c columnFamilyName C 2024-11-10T19:17:42,458 DEBUG [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:42,458 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(327): Store=bccf9fccd1002d9d7579e690ca8d056c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:17:42,458 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:42,459 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,459 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,460 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,461 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,461 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,461 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:17:42,462 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,464 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:17:42,465 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened bccf9fccd1002d9d7579e690ca8d056c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58747468, jitterRate=-0.12459450960159302}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:17:42,465 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:42,466 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for bccf9fccd1002d9d7579e690ca8d056c: Running coprocessor pre-open hook at 1731266262451Writing region info on filesystem at 1731266262451Initializing all the Stores at 1731266262452 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266262452Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266262452Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266262452Cleaning up temporary data from old regions at 1731266262461 (+9 ms)Running coprocessor post-open hooks at 1731266262465 (+4 ms)Region opened successfully at 1731266262465 2024-11-10T19:17:42,466 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., pid=81, masterSystemTime=1731266262447 2024-11-10T19:17:42,468 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:42,468 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:42,469 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=bccf9fccd1002d9d7579e690ca8d056c, regionState=OPEN, openSeqNum=2, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:42,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:17:42,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T19:17:42,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 in 176 msec 2024-11-10T19:17:42,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-10T19:17:42,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, ASSIGN in 334 msec 2024-11-10T19:17:42,475 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T19:17:42,475 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266262475"}]},"ts":"1731266262475"} 2024-11-10T19:17:42,477 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T19:17:42,477 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T19:17:42,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2010 sec 2024-11-10T19:17:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-10T19:17:43,432 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:43,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:43,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:43,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:43,434 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc36677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:43,436 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7432dacc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:43,438 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:43,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:43,441 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T19:17:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bdc25462c9ef,46761,1731266166843 2024-11-10T19:17:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@73b0bac0 2024-11-10T19:17:43,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T19:17:43,442 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T19:17:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T19:17:43,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:17:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T19:17:43,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742170_1346 (size=1097) 2024-11-10T19:17:43,878 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T19:17:43,878 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T19:17:43,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T19:17:43,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, REOPEN/MOVE}] 2024-11-10T19:17:43,884 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, REOPEN/MOVE 2024-11-10T19:17:43,884 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=bccf9fccd1002d9d7579e690ca8d056c, regionState=CLOSING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:43,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, REOPEN/MOVE because future has completed 2024-11-10T19:17:43,886 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T19:17:43,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:17:44,039 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,039 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T19:17:44,039 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing bccf9fccd1002d9d7579e690ca8d056c, disabling compactions & flushes 2024-11-10T19:17:44,039 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,039 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,040 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. after waiting 0 ms 2024-11-10T19:17:44,040 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,044 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T19:17:44,045 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,045 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for bccf9fccd1002d9d7579e690ca8d056c: Waiting for close lock at 1731266264039Running coprocessor pre-close hooks at 1731266264039Disabling compacts and flushes for region at 1731266264039Disabling writes for close at 1731266264040 (+1 ms)Writing region close event to WAL at 1731266264040Running coprocessor post-close hooks at 1731266264044 (+4 ms)Closed at 1731266264044 2024-11-10T19:17:44,045 WARN [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: bccf9fccd1002d9d7579e690ca8d056c to self. 2024-11-10T19:17:44,047 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,047 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=bccf9fccd1002d9d7579e690ca8d056c, regionState=CLOSED 2024-11-10T19:17:44,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:17:44,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-10T19:17:44,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 in 170 msec 2024-11-10T19:17:44,058 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, REOPEN/MOVE; state=CLOSED, location=bdc25462c9ef,34487,1731266167729; forceNewPlan=false, retain=true 2024-11-10T19:17:44,209 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=bccf9fccd1002d9d7579e690ca8d056c, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, REOPEN/MOVE because future has completed 2024-11-10T19:17:44,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:17:44,365 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,365 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:17:44,366 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,366 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:17:44,366 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,366 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,367 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,368 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:17:44,368 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bccf9fccd1002d9d7579e690ca8d056c columnFamilyName A 2024-11-10T19:17:44,369 DEBUG [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:44,370 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(327): Store=bccf9fccd1002d9d7579e690ca8d056c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:17:44,370 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,371 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:17:44,371 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bccf9fccd1002d9d7579e690ca8d056c columnFamilyName B 2024-11-10T19:17:44,371 DEBUG [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:44,372 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(327): Store=bccf9fccd1002d9d7579e690ca8d056c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:17:44,372 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,372 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:17:44,373 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bccf9fccd1002d9d7579e690ca8d056c columnFamilyName C 2024-11-10T19:17:44,373 DEBUG [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:44,373 INFO [StoreOpener-bccf9fccd1002d9d7579e690ca8d056c-1 {}] regionserver.HStore(327): Store=bccf9fccd1002d9d7579e690ca8d056c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:17:44,373 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,373 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,374 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,375 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,375 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,375 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,376 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:17:44,377 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,377 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened bccf9fccd1002d9d7579e690ca8d056c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66220976, jitterRate=-0.013230562210083008}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:17:44,378 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,378 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for bccf9fccd1002d9d7579e690ca8d056c: Running coprocessor pre-open hook at 1731266264366Writing region info on filesystem at 1731266264366Initializing all the Stores at 1731266264367 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266264367Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266264367Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266264367Cleaning up temporary data from old regions at 1731266264375 (+8 ms)Running coprocessor post-open hooks at 1731266264378 (+3 ms)Region opened successfully at 1731266264378 2024-11-10T19:17:44,379 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., pid=86, masterSystemTime=1731266264362 2024-11-10T19:17:44,381 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,381 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,382 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=bccf9fccd1002d9d7579e690ca8d056c, regionState=OPEN, openSeqNum=5, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:17:44,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-10T19:17:44,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 in 174 msec 2024-11-10T19:17:44,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-10T19:17:44,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, REOPEN/MOVE in 504 msec 2024-11-10T19:17:44,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T19:17:44,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-10T19:17:44,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 948 msec 2024-11-10T19:17:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T19:17:44,396 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:44,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,398 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0b6890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,400 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d761e9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,402 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba11d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b54b72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d31ddfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,408 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a9da84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,413 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d30ad27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,415 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b6d23a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,417 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d496b75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,419 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3190f54d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,426 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73510cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,428 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74469dd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,434 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2141fa68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,436 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8b0fa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,439 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bec55a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,442 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0b6e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d56d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d0bc02e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:17:44,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:17:44,454 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:17:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:17:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:17:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d78d91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:17:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:17:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:17:44,465 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:17:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f62976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:17:44,466 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,466 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,466 DEBUG [Thread-1591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,467 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,467 DEBUG [Thread-1589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,468 DEBUG [Thread-1580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-10T19:17:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T19:17:44,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,475 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,477 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:44,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:44,478 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,479 DEBUG [Thread-1587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,479 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:17:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,494 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:17:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:17:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:17:44,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:17:44,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:17:44,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:44,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101cda226582484e57a65a0be532237ff1_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266264486/Put/seqid=0 2024-11-10T19:17:44,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266324537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266324538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266324539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266324556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266324558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T19:17:44,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742171_1347 (size=14594) 2024-11-10T19:17:44,584 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:44,589 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101cda226582484e57a65a0be532237ff1_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101cda226582484e57a65a0be532237ff1_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:44,590 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/03e908bccb6c494db0b57eaeaf1035ae, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:44,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/03e908bccb6c494db0b57eaeaf1035ae is 175, key is test_row_0/A:col10/1731266264486/Put/seqid=0 2024-11-10T19:17:44,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742172_1348 (size=39549) 2024-11-10T19:17:44,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:44,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:44,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266324661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266324661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266324661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266324682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266324682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T19:17:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266324871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266324871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266324871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266324891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:44,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266324892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:44,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:44,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:44,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,003 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/03e908bccb6c494db0b57eaeaf1035ae 2024-11-10T19:17:45,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/8800ee97b0e54591ad69fdc6263fe4e8 is 50, key is test_row_0/B:col10/1731266264486/Put/seqid=0 2024-11-10T19:17:45,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742173_1349 (size=12001) 2024-11-10T19:17:45,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/8800ee97b0e54591ad69fdc6263fe4e8 2024-11-10T19:17:45,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/35690ec2556c4da687cf415f8d83bb72 is 50, key is test_row_0/C:col10/1731266264486/Put/seqid=0 2024-11-10T19:17:45,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:45,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T19:17:45,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742174_1350 (size=12001) 2024-11-10T19:17:45,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/35690ec2556c4da687cf415f8d83bb72 2024-11-10T19:17:45,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/03e908bccb6c494db0b57eaeaf1035ae as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/03e908bccb6c494db0b57eaeaf1035ae 2024-11-10T19:17:45,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/03e908bccb6c494db0b57eaeaf1035ae, entries=200, sequenceid=16, filesize=38.6 K 2024-11-10T19:17:45,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/8800ee97b0e54591ad69fdc6263fe4e8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/8800ee97b0e54591ad69fdc6263fe4e8 2024-11-10T19:17:45,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/8800ee97b0e54591ad69fdc6263fe4e8, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T19:17:45,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/35690ec2556c4da687cf415f8d83bb72 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/35690ec2556c4da687cf415f8d83bb72 2024-11-10T19:17:45,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/35690ec2556c4da687cf415f8d83bb72, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T19:17:45,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bccf9fccd1002d9d7579e690ca8d056c in 640ms, sequenceid=16, compaction requested=false 2024-11-10T19:17:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T19:17:45,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:45,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T19:17:45,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:45,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:45,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:45,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:45,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:45,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:45,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ffe29aa02600413bafc8cad0ccd1d70b_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266264551/Put/seqid=0 2024-11-10T19:17:45,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742175_1351 (size=14594) 2024-11-10T19:17:45,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266325201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266325201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266325202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266325202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266325211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:45,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266325322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266325322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266325322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:45,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:45,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266325542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266325542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266325542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:45,610 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ffe29aa02600413bafc8cad0ccd1d70b_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ffe29aa02600413bafc8cad0ccd1d70b_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T19:17:45,614 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/552886479e254fe28c15414ea1afad1d, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:45,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/552886479e254fe28c15414ea1afad1d is 175, key is test_row_0/A:col10/1731266264551/Put/seqid=0 2024-11-10T19:17:45,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742176_1352 (size=39549) 2024-11-10T19:17:45,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=37, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/552886479e254fe28c15414ea1afad1d 2024-11-10T19:17:45,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/15def9702fe140c98de9c81c8a3c4fc6 is 50, key is test_row_0/B:col10/1731266264551/Put/seqid=0 2024-11-10T19:17:45,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742177_1353 (size=12001) 2024-11-10T19:17:45,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266325722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266325722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:45,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:45,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:45,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:45,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266325861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266325862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266325864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:46,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:46,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:46,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:46,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:46,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/15def9702fe140c98de9c81c8a3c4fc6 2024-11-10T19:17:46,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/8d0394067cc1476189a07a33afa0466f is 50, key is test_row_0/C:col10/1731266264551/Put/seqid=0 2024-11-10T19:17:46,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742178_1354 (size=12001) 2024-11-10T19:17:46,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:46,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:46,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:46,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:46,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,310 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T19:17:46,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:46,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:46,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:46,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:46,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266326371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:46,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266326371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:46,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266326382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:46,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:46,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:46,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:46,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:46,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:46,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/8d0394067cc1476189a07a33afa0466f 2024-11-10T19:17:46,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/552886479e254fe28c15414ea1afad1d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/552886479e254fe28c15414ea1afad1d 2024-11-10T19:17:46,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/552886479e254fe28c15414ea1afad1d, entries=200, sequenceid=37, filesize=38.6 K 2024-11-10T19:17:46,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/15def9702fe140c98de9c81c8a3c4fc6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/15def9702fe140c98de9c81c8a3c4fc6 2024-11-10T19:17:46,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/15def9702fe140c98de9c81c8a3c4fc6, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T19:17:46,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/8d0394067cc1476189a07a33afa0466f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/8d0394067cc1476189a07a33afa0466f 2024-11-10T19:17:46,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/8d0394067cc1476189a07a33afa0466f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T19:17:46,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for bccf9fccd1002d9d7579e690ca8d056c in 1347ms, sequenceid=37, compaction requested=false 2024-11-10T19:17:46,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T19:17:46,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-10T19:17:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:46,629 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T19:17:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:46,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:46,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:46,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110465140f7c3a94211acf851aba20de369_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266265209/Put/seqid=0 2024-11-10T19:17:46,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742179_1355 (size=12154) 2024-11-10T19:17:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:46,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:46,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266326832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:46,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266266815, max timestamp=1731266266844Num uniques 859; 2024-11-10T19:17:46,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:46,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266266815, max timestamp=1731266266844Num uniques 859; 2024-11-10T19:17:46,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:46,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266266815, max timestamp=1731266266844Num uniques 859; 2024-11-10T19:17:46,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266327015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266327017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:47,066 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110465140f7c3a94211acf851aba20de369_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110465140f7c3a94211acf851aba20de369_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:47,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/350ad61b808f410d9cc291b3bbf991aa, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:47,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/350ad61b808f410d9cc291b3bbf991aa is 175, key is test_row_0/A:col10/1731266265209/Put/seqid=0 2024-11-10T19:17:47,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742180_1356 (size=30955) 2024-11-10T19:17:47,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266327141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266327141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266327352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266327361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266327391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266327391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266327391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,401 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:47,401 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,402 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,478 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/350ad61b808f410d9cc291b3bbf991aa 2024-11-10T19:17:47,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/be5951ad96de4bd29275df2cffc38cdf is 50, key is test_row_0/B:col10/1731266265209/Put/seqid=0 2024-11-10T19:17:47,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742181_1357 (size=12001) 2024-11-10T19:17:47,529 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/be5951ad96de4bd29275df2cffc38cdf 2024-11-10T19:17:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/78b275f63f064e6fb42330b332bed207 is 50, key is test_row_0/C:col10/1731266265209/Put/seqid=0 2024-11-10T19:17:47,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742182_1358 (size=12001) 2024-11-10T19:17:47,572 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/78b275f63f064e6fb42330b332bed207 2024-11-10T19:17:47,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/350ad61b808f410d9cc291b3bbf991aa as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/350ad61b808f410d9cc291b3bbf991aa 2024-11-10T19:17:47,583 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/350ad61b808f410d9cc291b3bbf991aa, entries=150, sequenceid=48, filesize=30.2 K 2024-11-10T19:17:47,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/be5951ad96de4bd29275df2cffc38cdf as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be5951ad96de4bd29275df2cffc38cdf 2024-11-10T19:17:47,593 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be5951ad96de4bd29275df2cffc38cdf, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T19:17:47,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/78b275f63f064e6fb42330b332bed207 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/78b275f63f064e6fb42330b332bed207 2024-11-10T19:17:47,597 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/78b275f63f064e6fb42330b332bed207, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T19:17:47,604 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bccf9fccd1002d9d7579e690ca8d056c in 975ms, sequenceid=48, compaction requested=true 2024-11-10T19:17:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-10T19:17:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-10T19:17:47,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-10T19:17:47,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1310 sec 2024-11-10T19:17:47,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.1390 sec 2024-11-10T19:17:47,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T19:17:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:47,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110be59d6ac1dc24ce5bd3d62706f480e98_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266267015/Put/seqid=0 2024-11-10T19:17:47,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742183_1359 (size=14594) 2024-11-10T19:17:47,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266327734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266327735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266327854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266327861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266328071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266328081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:48,112 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:48,117 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110be59d6ac1dc24ce5bd3d62706f480e98_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be59d6ac1dc24ce5bd3d62706f480e98_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:48,117 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/70263388620e4c53b69fb2e51e915892, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:48,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/70263388620e4c53b69fb2e51e915892 is 175, key is test_row_0/A:col10/1731266267015/Put/seqid=0 2024-11-10T19:17:48,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742184_1360 (size=39549) 2024-11-10T19:17:48,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266328391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266328401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:48,521 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/70263388620e4c53b69fb2e51e915892 2024-11-10T19:17:48,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/ce3460c18db64a8fa5ec296e1036a7d9 is 50, key is test_row_0/B:col10/1731266267015/Put/seqid=0 2024-11-10T19:17:48,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742185_1361 (size=12001) 2024-11-10T19:17:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T19:17:48,632 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:48,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-10T19:17:48,634 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:48,635 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:48,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T19:17:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T19:17:48,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-10T19:17:48,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:48,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:48,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:48,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:48,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:48,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266328911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:48,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266328922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:48,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-10T19:17:48,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:48,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:48,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:48,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:48,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:48,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/ce3460c18db64a8fa5ec296e1036a7d9 2024-11-10T19:17:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T19:17:48,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/f1d7817656bd4adbbc76ffa922138b34 is 50, key is test_row_0/C:col10/1731266267015/Put/seqid=0 2024-11-10T19:17:48,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742186_1362 (size=12001) 2024-11-10T19:17:48,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/f1d7817656bd4adbbc76ffa922138b34 2024-11-10T19:17:48,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/70263388620e4c53b69fb2e51e915892 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/70263388620e4c53b69fb2e51e915892 2024-11-10T19:17:48,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/70263388620e4c53b69fb2e51e915892, entries=200, sequenceid=85, filesize=38.6 K 2024-11-10T19:17:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/ce3460c18db64a8fa5ec296e1036a7d9 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ce3460c18db64a8fa5ec296e1036a7d9 2024-11-10T19:17:48,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ce3460c18db64a8fa5ec296e1036a7d9, entries=150, sequenceid=85, filesize=11.7 K 2024-11-10T19:17:48,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/f1d7817656bd4adbbc76ffa922138b34 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f1d7817656bd4adbbc76ffa922138b34 2024-11-10T19:17:49,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f1d7817656bd4adbbc76ffa922138b34, entries=150, sequenceid=85, filesize=11.7 K 2024-11-10T19:17:49,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for bccf9fccd1002d9d7579e690ca8d056c in 1326ms, sequenceid=85, compaction requested=true 2024-11-10T19:17:49,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:49,003 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:49,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:49,008 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:49,008 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/A is initiating minor compaction (all files) 2024-11-10T19:17:49,008 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/A in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:49,008 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/03e908bccb6c494db0b57eaeaf1035ae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/552886479e254fe28c15414ea1afad1d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/350ad61b808f410d9cc291b3bbf991aa, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/70263388620e4c53b69fb2e51e915892] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=146.1 K 2024-11-10T19:17:49,008 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:49,009 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/03e908bccb6c494db0b57eaeaf1035ae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/552886479e254fe28c15414ea1afad1d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/350ad61b808f410d9cc291b3bbf991aa, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/70263388620e4c53b69fb2e51e915892] 2024-11-10T19:17:49,009 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e908bccb6c494db0b57eaeaf1035ae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731266264485 2024-11-10T19:17:49,009 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 552886479e254fe28c15414ea1afad1d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731266264530 2024-11-10T19:17:49,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:49,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:49,010 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:49,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:49,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:49,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:49,010 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 350ad61b808f410d9cc291b3bbf991aa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731266265196 2024-11-10T19:17:49,010 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70263388620e4c53b69fb2e51e915892, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731266266953 2024-11-10T19:17:49,011 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:49,011 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/B is initiating minor compaction (all files) 2024-11-10T19:17:49,011 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/B in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:49,011 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/8800ee97b0e54591ad69fdc6263fe4e8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/15def9702fe140c98de9c81c8a3c4fc6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be5951ad96de4bd29275df2cffc38cdf, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ce3460c18db64a8fa5ec296e1036a7d9] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=46.9 K 2024-11-10T19:17:49,012 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 8800ee97b0e54591ad69fdc6263fe4e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731266264486 2024-11-10T19:17:49,012 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 15def9702fe140c98de9c81c8a3c4fc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731266264530 2024-11-10T19:17:49,012 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting be5951ad96de4bd29275df2cffc38cdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731266265196 2024-11-10T19:17:49,013 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3460c18db64a8fa5ec296e1036a7d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731266266975 2024-11-10T19:17:49,019 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:49,022 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:49,023 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/9f6c23e66fcc49b4a46cac77ac67d9e5 is 50, key is test_row_0/B:col10/1731266267015/Put/seqid=0 2024-11-10T19:17:49,024 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111081e9ba3b79b94deeab6c9a24495ea563_bccf9fccd1002d9d7579e690ca8d056c store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:49,027 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111081e9ba3b79b94deeab6c9a24495ea563_bccf9fccd1002d9d7579e690ca8d056c, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:49,027 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111081e9ba3b79b94deeab6c9a24495ea563_bccf9fccd1002d9d7579e690ca8d056c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:49,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742187_1363 (size=12139) 2024-11-10T19:17:49,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742188_1364 (size=4469) 2024-11-10T19:17:49,078 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#A#compaction#306 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:49,079 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/2ff33630c67c477fb572563bb3774b9d is 175, key is test_row_0/A:col10/1731266267015/Put/seqid=0 2024-11-10T19:17:49,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-10T19:17:49,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:49,101 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:17:49,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:49,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:49,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:49,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:49,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:49,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:49,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742189_1365 (size=31093) 2024-11-10T19:17:49,116 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/2ff33630c67c477fb572563bb3774b9d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2ff33630c67c477fb572563bb3774b9d 2024-11-10T19:17:49,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111044fb7d7cb6ae4b2fac2e758f0ea35072_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266267717/Put/seqid=0 2024-11-10T19:17:49,121 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/A of bccf9fccd1002d9d7579e690ca8d056c into 2ff33630c67c477fb572563bb3774b9d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:49,121 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:49,121 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/A, priority=12, startTime=1731266269003; duration=0sec 2024-11-10T19:17:49,122 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:49,122 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:A 2024-11-10T19:17:49,122 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:49,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742190_1366 (size=12154) 2024-11-10T19:17:49,125 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:49,125 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/C is initiating minor compaction (all files) 2024-11-10T19:17:49,125 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/C in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:49,125 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/35690ec2556c4da687cf415f8d83bb72, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/8d0394067cc1476189a07a33afa0466f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/78b275f63f064e6fb42330b332bed207, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f1d7817656bd4adbbc76ffa922138b34] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=46.9 K 2024-11-10T19:17:49,126 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35690ec2556c4da687cf415f8d83bb72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731266264486 2024-11-10T19:17:49,126 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d0394067cc1476189a07a33afa0466f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731266264530 2024-11-10T19:17:49,126 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78b275f63f064e6fb42330b332bed207, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731266265196 2024-11-10T19:17:49,127 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1d7817656bd4adbbc76ffa922138b34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731266266975 2024-11-10T19:17:49,141 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:49,142 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/3a76b245b3e7487facdc6b95b1ee9f68 is 50, key is test_row_0/C:col10/1731266267015/Put/seqid=0 2024-11-10T19:17:49,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742191_1367 (size=12139) 2024-11-10T19:17:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T19:17:49,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:49,463 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/9f6c23e66fcc49b4a46cac77ac67d9e5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/9f6c23e66fcc49b4a46cac77ac67d9e5 2024-11-10T19:17:49,470 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/B of bccf9fccd1002d9d7579e690ca8d056c into 9f6c23e66fcc49b4a46cac77ac67d9e5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:49,470 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:49,470 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/B, priority=12, startTime=1731266269009; duration=0sec 2024-11-10T19:17:49,470 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:49,470 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:B 2024-11-10T19:17:49,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:49,529 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111044fb7d7cb6ae4b2fac2e758f0ea35072_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111044fb7d7cb6ae4b2fac2e758f0ea35072_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5708f557702a4958b72ac4771eae55a4, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5708f557702a4958b72ac4771eae55a4 is 175, key is test_row_0/A:col10/1731266267717/Put/seqid=0 2024-11-10T19:17:49,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266269534, max timestamp=1731266269556Num uniques 859; 2024-11-10T19:17:49,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:49,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742192_1368 (size=30955) 2024-11-10T19:17:49,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266269534, max timestamp=1731266269559Num uniques 859; 2024-11-10T19:17:49,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:49,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731266269534, max timestamp=1731266269556Num uniques 859; 2024-11-10T19:17:49,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T19:17:49,599 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/3a76b245b3e7487facdc6b95b1ee9f68 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3a76b245b3e7487facdc6b95b1ee9f68 2024-11-10T19:17:49,607 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/C of bccf9fccd1002d9d7579e690ca8d056c into 3a76b245b3e7487facdc6b95b1ee9f68(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:49,607 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:49,607 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/C, priority=12, startTime=1731266269010; duration=0sec 2024-11-10T19:17:49,607 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:49,607 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:C 2024-11-10T19:17:49,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266329661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266329669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:49,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266329671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T19:17:49,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266329782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:49,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266329792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:49,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266329793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:49,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266329941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:49,948 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:49,960 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5708f557702a4958b72ac4771eae55a4 2024-11-10T19:17:49,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266329951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:49,966 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:49,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/5327f33485bc43c4adc29d2a7dbb271f is 50, key is test_row_0/B:col10/1731266267717/Put/seqid=0 2024-11-10T19:17:50,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266330003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266330014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266330014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742193_1369 (size=12001) 2024-11-10T19:17:50,033 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/5327f33485bc43c4adc29d2a7dbb271f 2024-11-10T19:17:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/b15e372204da4e59bff4fde6dddad7ef is 50, key is test_row_0/C:col10/1731266267717/Put/seqid=0 2024-11-10T19:17:50,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742194_1370 (size=12001) 2024-11-10T19:17:50,069 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/b15e372204da4e59bff4fde6dddad7ef 2024-11-10T19:17:50,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5708f557702a4958b72ac4771eae55a4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5708f557702a4958b72ac4771eae55a4 2024-11-10T19:17:50,095 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5708f557702a4958b72ac4771eae55a4, entries=150, sequenceid=95, filesize=30.2 K 2024-11-10T19:17:50,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/5327f33485bc43c4adc29d2a7dbb271f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5327f33485bc43c4adc29d2a7dbb271f 2024-11-10T19:17:50,104 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5327f33485bc43c4adc29d2a7dbb271f, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T19:17:50,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/b15e372204da4e59bff4fde6dddad7ef as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/b15e372204da4e59bff4fde6dddad7ef 2024-11-10T19:17:50,108 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/b15e372204da4e59bff4fde6dddad7ef, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T19:17:50,112 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for bccf9fccd1002d9d7579e690ca8d056c in 1011ms, sequenceid=95, compaction requested=false 2024-11-10T19:17:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-10T19:17:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-10T19:17:50,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-10T19:17:50,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-11-10T19:17:50,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.4860 sec 2024-11-10T19:17:50,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=140.49 KB heapSize=404.70 KB 2024-11-10T19:17:50,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:50,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T19:17:50,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:50,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T19:17:50,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:50,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T19:17:50,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:50,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b09d583be3f64f3dbbc2ddeba3fa3ec1_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266269665/Put/seqid=0 2024-11-10T19:17:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266330357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266330359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266330381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742195_1371 (size=14794) 2024-11-10T19:17:50,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266330491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266330491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266330491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266330712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266330712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266330724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:50,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T19:17:50,794 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:50,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-10T19:17:50,797 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:50,798 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:50,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T19:17:50,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:50,820 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b09d583be3f64f3dbbc2ddeba3fa3ec1_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b09d583be3f64f3dbbc2ddeba3fa3ec1_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:50,821 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/af485d6d11d24510b3583d64ad7f7ccc, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:50,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/af485d6d11d24510b3583d64ad7f7ccc is 175, key is test_row_0/A:col10/1731266269665/Put/seqid=0 2024-11-10T19:17:50,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742196_1372 (size=39749) 2024-11-10T19:17:50,867 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/af485d6d11d24510b3583d64ad7f7ccc 2024-11-10T19:17:50,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/231359ca583e450b8eda6f2dbb2561f3 is 50, key is test_row_0/B:col10/1731266269665/Put/seqid=0 2024-11-10T19:17:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T19:17:50,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742197_1373 (size=12151) 2024-11-10T19:17:50,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/231359ca583e450b8eda6f2dbb2561f3 2024-11-10T19:17:50,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:50,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:50,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/c0011768d7f049f388d7fcf7291aad0e is 50, key is test_row_0/C:col10/1731266269665/Put/seqid=0 2024-11-10T19:17:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:50,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742198_1374 (size=12151) 2024-11-10T19:17:50,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/c0011768d7f049f388d7fcf7291aad0e 2024-11-10T19:17:51,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/af485d6d11d24510b3583d64ad7f7ccc as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/af485d6d11d24510b3583d64ad7f7ccc 2024-11-10T19:17:51,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/af485d6d11d24510b3583d64ad7f7ccc, entries=200, sequenceid=136, filesize=38.8 K 2024-11-10T19:17:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/231359ca583e450b8eda6f2dbb2561f3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/231359ca583e450b8eda6f2dbb2561f3 2024-11-10T19:17:51,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/231359ca583e450b8eda6f2dbb2561f3, entries=150, sequenceid=136, filesize=11.9 K 2024-11-10T19:17:51,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/c0011768d7f049f388d7fcf7291aad0e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c0011768d7f049f388d7fcf7291aad0e 2024-11-10T19:17:51,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c0011768d7f049f388d7fcf7291aad0e, entries=150, sequenceid=136, filesize=11.9 K 2024-11-10T19:17:51,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~404.65 KB/414360, currentSize=40.25 KB/41220 for bccf9fccd1002d9d7579e690ca8d056c in 690ms, sequenceid=136, compaction requested=true 2024-11-10T19:17:51,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:51,022 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:51,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:51,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:51,023 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:51,023 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:51,023 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/A is initiating minor compaction (all files) 2024-11-10T19:17:51,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:51,023 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/A in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:51,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:51,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:51,023 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2ff33630c67c477fb572563bb3774b9d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5708f557702a4958b72ac4771eae55a4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/af485d6d11d24510b3583d64ad7f7ccc] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=99.4 K 2024-11-10T19:17:51,023 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,024 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2ff33630c67c477fb572563bb3774b9d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5708f557702a4958b72ac4771eae55a4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/af485d6d11d24510b3583d64ad7f7ccc] 2024-11-10T19:17:51,024 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ff33630c67c477fb572563bb3774b9d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731266266975 2024-11-10T19:17:51,024 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5708f557702a4958b72ac4771eae55a4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731266267717 2024-11-10T19:17:51,025 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting af485d6d11d24510b3583d64ad7f7ccc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731266269641 2024-11-10T19:17:51,027 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:51,027 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/B is initiating minor compaction (all files) 2024-11-10T19:17:51,027 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/B in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,027 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/9f6c23e66fcc49b4a46cac77ac67d9e5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5327f33485bc43c4adc29d2a7dbb271f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/231359ca583e450b8eda6f2dbb2561f3] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=35.4 K 2024-11-10T19:17:51,027 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f6c23e66fcc49b4a46cac77ac67d9e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731266266975 2024-11-10T19:17:51,028 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5327f33485bc43c4adc29d2a7dbb271f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731266267717 2024-11-10T19:17:51,028 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 231359ca583e450b8eda6f2dbb2561f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731266269665 2024-11-10T19:17:51,037 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:51,042 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#B#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:51,043 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/2565ab1fdfa04e84aac37fdbcb6e13b8 is 50, key is test_row_0/B:col10/1731266269665/Put/seqid=0 2024-11-10T19:17:51,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:17:51,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:51,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:51,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:51,051 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110336dfa11b2694bbc9eaa9b1f88d9981a_bccf9fccd1002d9d7579e690ca8d056c store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:51,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:51,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:51,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:51,053 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110336dfa11b2694bbc9eaa9b1f88d9981a_bccf9fccd1002d9d7579e690ca8d056c, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:51,054 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110336dfa11b2694bbc9eaa9b1f88d9981a_bccf9fccd1002d9d7579e690ca8d056c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T19:17:51,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742199_1375 (size=12391) 2024-11-10T19:17:51,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111021df7ce12d5649729c2bcd4a133da3c4_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266270379/Put/seqid=0 2024-11-10T19:17:51,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742200_1376 (size=4469) 2024-11-10T19:17:51,175 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#A#compaction#315 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:51,176 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/f9f1ffb538fb484784a894ba0eff8f0b is 175, key is test_row_0/A:col10/1731266269665/Put/seqid=0 2024-11-10T19:17:51,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742201_1377 (size=14794) 2024-11-10T19:17:51,181 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:51,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111021df7ce12d5649729c2bcd4a133da3c4_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021df7ce12d5649729c2bcd4a133da3c4_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:51,188 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/6a61d9ce3dee4c27bf9e82363c932958, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:51,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/6a61d9ce3dee4c27bf9e82363c932958 is 175, key is test_row_0/A:col10/1731266270379/Put/seqid=0 2024-11-10T19:17:51,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731266271147, max timestamp=1731266271206Num uniques 859; 2024-11-10T19:17:51,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T19:17:51,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742202_1378 (size=31345) 2024-11-10T19:17:51,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266331254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742203_1379 (size=39749) 2024-11-10T19:17:51,260 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/f9f1ffb538fb484784a894ba0eff8f0b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f9f1ffb538fb484784a894ba0eff8f0b 2024-11-10T19:17:51,266 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/A of bccf9fccd1002d9d7579e690ca8d056c into f9f1ffb538fb484784a894ba0eff8f0b(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:51,266 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:51,266 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/A, priority=13, startTime=1731266271022; duration=0sec 2024-11-10T19:17:51,266 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:51,266 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:A 2024-11-10T19:17:51,266 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:51,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:51,268 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:51,268 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/C is initiating minor compaction (all files) 2024-11-10T19:17:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,269 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/C in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,269 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3a76b245b3e7487facdc6b95b1ee9f68, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/b15e372204da4e59bff4fde6dddad7ef, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c0011768d7f049f388d7fcf7291aad0e] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=35.4 K 2024-11-10T19:17:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,270 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a76b245b3e7487facdc6b95b1ee9f68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731266266975 2024-11-10T19:17:51,271 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting b15e372204da4e59bff4fde6dddad7ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731266267717 2024-11-10T19:17:51,272 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0011768d7f049f388d7fcf7291aad0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731266269665 2024-11-10T19:17:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266331256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266331265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,286 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:51,287 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/37b3fab531e54604aa0239243c41e76d is 50, key is test_row_0/C:col10/1731266269665/Put/seqid=0 2024-11-10T19:17:51,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742204_1380 (size=12391) 2024-11-10T19:17:51,301 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/37b3fab531e54604aa0239243c41e76d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/37b3fab531e54604aa0239243c41e76d 2024-11-10T19:17:51,306 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/C of bccf9fccd1002d9d7579e690ca8d056c into 37b3fab531e54604aa0239243c41e76d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:51,306 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:51,306 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/C, priority=13, startTime=1731266271023; duration=0sec 2024-11-10T19:17:51,306 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:51,306 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:C 2024-11-10T19:17:51,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266331361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266331391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266331391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T19:17:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,518 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/2565ab1fdfa04e84aac37fdbcb6e13b8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/2565ab1fdfa04e84aac37fdbcb6e13b8 2024-11-10T19:17:51,527 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/B of bccf9fccd1002d9d7579e690ca8d056c into 2565ab1fdfa04e84aac37fdbcb6e13b8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:51,527 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:51,527 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/B, priority=13, startTime=1731266271023; duration=0sec 2024-11-10T19:17:51,527 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:51,528 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:B 2024-11-10T19:17:51,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266331581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266331611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266331611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,668 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/6a61d9ce3dee4c27bf9e82363c932958 2024-11-10T19:17:51,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/a7e2ee2f238648c98780dc19bf52f9e8 is 50, key is test_row_0/B:col10/1731266270379/Put/seqid=0 2024-11-10T19:17:51,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:51,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742205_1381 (size=12151) 2024-11-10T19:17:51,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:51,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:51,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266331901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T19:17:51,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266331932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266331932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266331971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,985 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:51,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266331982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:51,994 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:52,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:52,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:52,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:52,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:52,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:52,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/a7e2ee2f238648c98780dc19bf52f9e8 2024-11-10T19:17:52,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/ca036378935141338af61f03e29cbbd6 is 50, key is test_row_0/C:col10/1731266270379/Put/seqid=0 2024-11-10T19:17:52,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:52,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:52,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742206_1382 (size=12151) 2024-11-10T19:17:52,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/ca036378935141338af61f03e29cbbd6 2024-11-10T19:17:52,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/6a61d9ce3dee4c27bf9e82363c932958 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/6a61d9ce3dee4c27bf9e82363c932958 2024-11-10T19:17:52,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/6a61d9ce3dee4c27bf9e82363c932958, entries=200, sequenceid=146, filesize=38.8 K 2024-11-10T19:17:52,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/a7e2ee2f238648c98780dc19bf52f9e8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/a7e2ee2f238648c98780dc19bf52f9e8 2024-11-10T19:17:52,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/a7e2ee2f238648c98780dc19bf52f9e8, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T19:17:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/ca036378935141338af61f03e29cbbd6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ca036378935141338af61f03e29cbbd6 2024-11-10T19:17:52,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ca036378935141338af61f03e29cbbd6, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T19:17:52,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for bccf9fccd1002d9d7579e690ca8d056c in 1207ms, sequenceid=146, compaction requested=false 2024-11-10T19:17:52,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:52,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-10T19:17:52,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:52,360 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=127.47 KB heapSize=371.88 KB 2024-11-10T19:17:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:52,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111046dd9dfd100f4f08902f987a76c923a7_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266271227/Put/seqid=0 2024-11-10T19:17:52,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742207_1383 (size=12304) 2024-11-10T19:17:52,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:52,409 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111046dd9dfd100f4f08902f987a76c923a7_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046dd9dfd100f4f08902f987a76c923a7_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:52,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/433c0e0225bf48beab5a0a834f203750, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:52,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/433c0e0225bf48beab5a0a834f203750 is 175, key is test_row_0/A:col10/1731266271227/Put/seqid=0 2024-11-10T19:17:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:52,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:52,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742208_1384 (size=31105) 2024-11-10T19:17:52,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266332468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266332470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266332476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266332591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266332592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266332601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266332812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266332812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266332822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:52,840 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/433c0e0225bf48beab5a0a834f203750 2024-11-10T19:17:52,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/be7ab7f723fd41ebab1657bd472d53b7 is 50, key is test_row_0/B:col10/1731266271227/Put/seqid=0 2024-11-10T19:17:52,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742209_1385 (size=12151) 2024-11-10T19:17:52,870 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/be7ab7f723fd41ebab1657bd472d53b7 2024-11-10T19:17:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/ae479fe6de4e45f990538e15716c7215 is 50, key is test_row_0/C:col10/1731266271227/Put/seqid=0 2024-11-10T19:17:52,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742210_1386 (size=12151) 2024-11-10T19:17:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T19:17:53,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266333132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:53,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266333132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:53,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266333142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:53,297 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/ae479fe6de4e45f990538e15716c7215 2024-11-10T19:17:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/433c0e0225bf48beab5a0a834f203750 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/433c0e0225bf48beab5a0a834f203750 2024-11-10T19:17:53,325 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/433c0e0225bf48beab5a0a834f203750, entries=150, sequenceid=175, filesize=30.4 K 2024-11-10T19:17:53,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/be7ab7f723fd41ebab1657bd472d53b7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be7ab7f723fd41ebab1657bd472d53b7 2024-11-10T19:17:53,353 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be7ab7f723fd41ebab1657bd472d53b7, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T19:17:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/ae479fe6de4e45f990538e15716c7215 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae479fe6de4e45f990538e15716c7215 2024-11-10T19:17:53,365 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae479fe6de4e45f990538e15716c7215, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T19:17:53,366 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~371.83 KB/380752, currentSize=46.96 KB/48090 for bccf9fccd1002d9d7579e690ca8d056c in 1006ms, sequenceid=175, compaction requested=true 2024-11-10T19:17:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-10T19:17:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-10T19:17:53,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-10T19:17:53,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5700 sec 2024-11-10T19:17:53,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.5770 sec 2024-11-10T19:17:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:53,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T19:17:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:53,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104464676aa5974b75b8eeb2f78256903b_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266272472/Put/seqid=0 2024-11-10T19:17:53,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742211_1387 (size=14794) 2024-11-10T19:17:53,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266333750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:53,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266333751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:53,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731266273674, max timestamp=1731266273761Num uniques 859; 2024-11-10T19:17:53,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731266273674, max timestamp=1731266273761Num uniques 859; 2024-11-10T19:17:53,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731266273674, max timestamp=1731266273761Num uniques 859; 2024-11-10T19:17:53,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T19:17:53,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T19:17:53,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:53,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T19:17:53,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:53,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266333891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:53,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266333894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:53,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266333894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266334002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266334021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266334021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:54,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104464676aa5974b75b8eeb2f78256903b_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104464676aa5974b75b8eeb2f78256903b_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:54,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/f7fddb24ee7245fe9666674c8063a122, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:54,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/f7fddb24ee7245fe9666674c8063a122 is 175, key is test_row_0/A:col10/1731266272472/Put/seqid=0 2024-11-10T19:17:54,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742212_1388 (size=39749) 2024-11-10T19:17:54,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/f7fddb24ee7245fe9666674c8063a122 2024-11-10T19:17:54,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/67a78de9d6be4dffbf2c25274297a638 is 50, key is test_row_0/B:col10/1731266272472/Put/seqid=0 2024-11-10T19:17:54,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742213_1389 (size=12151) 2024-11-10T19:17:54,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/67a78de9d6be4dffbf2c25274297a638 2024-11-10T19:17:54,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4f66c5f8d6664eec881d05b8d4b332a1 is 50, key is test_row_0/C:col10/1731266272472/Put/seqid=0 2024-11-10T19:17:54,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742214_1390 (size=12151) 2024-11-10T19:17:54,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4f66c5f8d6664eec881d05b8d4b332a1 2024-11-10T19:17:54,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/f7fddb24ee7245fe9666674c8063a122 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f7fddb24ee7245fe9666674c8063a122 2024-11-10T19:17:54,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f7fddb24ee7245fe9666674c8063a122, entries=200, sequenceid=186, filesize=38.8 K 2024-11-10T19:17:54,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/67a78de9d6be4dffbf2c25274297a638 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/67a78de9d6be4dffbf2c25274297a638 2024-11-10T19:17:54,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/67a78de9d6be4dffbf2c25274297a638, entries=150, sequenceid=186, filesize=11.9 K 2024-11-10T19:17:54,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4f66c5f8d6664eec881d05b8d4b332a1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4f66c5f8d6664eec881d05b8d4b332a1 2024-11-10T19:17:54,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4f66c5f8d6664eec881d05b8d4b332a1, entries=150, sequenceid=186, filesize=11.9 K 2024-11-10T19:17:54,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for bccf9fccd1002d9d7579e690ca8d056c in 540ms, sequenceid=186, compaction requested=true 2024-11-10T19:17:54,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:54,204 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:54,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:54,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:54,209 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:54,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:54,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:54,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:54,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:54,213 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141948 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:54,213 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/A is initiating minor compaction (all files) 2024-11-10T19:17:54,213 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/A in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:54,213 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f9f1ffb538fb484784a894ba0eff8f0b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/6a61d9ce3dee4c27bf9e82363c932958, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/433c0e0225bf48beab5a0a834f203750, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f7fddb24ee7245fe9666674c8063a122] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=138.6 K 2024-11-10T19:17:54,213 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:54,213 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f9f1ffb538fb484784a894ba0eff8f0b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/6a61d9ce3dee4c27bf9e82363c932958, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/433c0e0225bf48beab5a0a834f203750, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f7fddb24ee7245fe9666674c8063a122] 2024-11-10T19:17:54,214 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting f9f1ffb538fb484784a894ba0eff8f0b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731266269665 2024-11-10T19:17:54,214 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a61d9ce3dee4c27bf9e82363c932958, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731266270334 2024-11-10T19:17:54,214 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:54,214 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/B is initiating minor compaction (all files) 2024-11-10T19:17:54,215 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/B in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:54,215 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/2565ab1fdfa04e84aac37fdbcb6e13b8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/a7e2ee2f238648c98780dc19bf52f9e8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be7ab7f723fd41ebab1657bd472d53b7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/67a78de9d6be4dffbf2c25274297a638] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=47.7 K 2024-11-10T19:17:54,215 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 433c0e0225bf48beab5a0a834f203750, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731266271226 2024-11-10T19:17:54,215 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2565ab1fdfa04e84aac37fdbcb6e13b8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731266269665 2024-11-10T19:17:54,215 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting f7fddb24ee7245fe9666674c8063a122, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731266272466 2024-11-10T19:17:54,215 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7e2ee2f238648c98780dc19bf52f9e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731266270334 2024-11-10T19:17:54,215 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7ab7f723fd41ebab1657bd472d53b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731266271226 2024-11-10T19:17:54,216 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67a78de9d6be4dffbf2c25274297a638, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731266272466 2024-11-10T19:17:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:54,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-10T19:17:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:54,244 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:54,253 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#B#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:54,254 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/76588007bf6449d4ae212c3fb8939e35 is 50, key is test_row_0/B:col10/1731266272472/Put/seqid=0 2024-11-10T19:17:54,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111077a3c90ca0fd41599a2015558dae3042_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266273840/Put/seqid=0 2024-11-10T19:17:54,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742215_1391 (size=12527) 2024-11-10T19:17:54,265 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110684f88a6363e452c85caab37eae85942_bccf9fccd1002d9d7579e690ca8d056c store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:54,268 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110684f88a6363e452c85caab37eae85942_bccf9fccd1002d9d7579e690ca8d056c, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:54,269 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110684f88a6363e452c85caab37eae85942_bccf9fccd1002d9d7579e690ca8d056c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:54,278 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/76588007bf6449d4ae212c3fb8939e35 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/76588007bf6449d4ae212c3fb8939e35 2024-11-10T19:17:54,284 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/B of bccf9fccd1002d9d7579e690ca8d056c into 76588007bf6449d4ae212c3fb8939e35(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:54,284 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:54,284 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/B, priority=12, startTime=1731266274208; duration=0sec 2024-11-10T19:17:54,284 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:54,284 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:B 2024-11-10T19:17:54,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,284 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266334262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266334275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,301 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:54,301 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/C is initiating minor compaction (all files) 2024-11-10T19:17:54,301 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/C in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:54,301 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/37b3fab531e54604aa0239243c41e76d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ca036378935141338af61f03e29cbbd6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae479fe6de4e45f990538e15716c7215, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4f66c5f8d6664eec881d05b8d4b332a1] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=47.7 K 2024-11-10T19:17:54,306 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b3fab531e54604aa0239243c41e76d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731266269665 2024-11-10T19:17:54,307 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca036378935141338af61f03e29cbbd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731266270334 2024-11-10T19:17:54,307 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae479fe6de4e45f990538e15716c7215, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731266271226 2024-11-10T19:17:54,307 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f66c5f8d6664eec881d05b8d4b332a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731266272466 2024-11-10T19:17:54,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266334291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742216_1392 (size=14794) 2024-11-10T19:17:54,318 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:54,328 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#C#compaction#330 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:54,333 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111077a3c90ca0fd41599a2015558dae3042_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111077a3c90ca0fd41599a2015558dae3042_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:54,333 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/e2f60d1f936b4c5698cdd38246124808 is 50, key is test_row_0/C:col10/1731266272472/Put/seqid=0 2024-11-10T19:17:54,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/9e89d099ee5b4f699049ebf54b1ef9c7, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:54,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/9e89d099ee5b4f699049ebf54b1ef9c7 is 175, key is test_row_0/A:col10/1731266273840/Put/seqid=0 2024-11-10T19:17:54,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742217_1393 (size=4469) 2024-11-10T19:17:54,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742218_1394 (size=12527) 2024-11-10T19:17:54,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742219_1395 (size=39749) 2024-11-10T19:17:54,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266334401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266334401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266334422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266334611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266334611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266334631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:54,758 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#A#compaction#327 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:54,759 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5ec8c0ada5e246d1b9331c2d6500a64b is 175, key is test_row_0/A:col10/1731266272472/Put/seqid=0 2024-11-10T19:17:54,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742220_1396 (size=31481) 2024-11-10T19:17:54,779 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/e2f60d1f936b4c5698cdd38246124808 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/e2f60d1f936b4c5698cdd38246124808 2024-11-10T19:17:54,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/9e89d099ee5b4f699049ebf54b1ef9c7 2024-11-10T19:17:54,785 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/C of bccf9fccd1002d9d7579e690ca8d056c into e2f60d1f936b4c5698cdd38246124808(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:54,785 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:54,785 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/C, priority=12, startTime=1731266274209; duration=0sec 2024-11-10T19:17:54,786 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:54,786 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:C 2024-11-10T19:17:54,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/3c05eb6f1f85486882dabea44f5f591a is 50, key is test_row_0/B:col10/1731266273840/Put/seqid=0 2024-11-10T19:17:54,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742221_1397 (size=12151) 2024-11-10T19:17:54,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/3c05eb6f1f85486882dabea44f5f591a 2024-11-10T19:17:54,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/0aaca40d3ece45d39d6fee1836130314 is 50, key is test_row_0/C:col10/1731266273840/Put/seqid=0 2024-11-10T19:17:54,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742222_1398 (size=12151) 2024-11-10T19:17:54,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/0aaca40d3ece45d39d6fee1836130314 2024-11-10T19:17:54,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/9e89d099ee5b4f699049ebf54b1ef9c7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9e89d099ee5b4f699049ebf54b1ef9c7 2024-11-10T19:17:54,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9e89d099ee5b4f699049ebf54b1ef9c7, entries=200, sequenceid=222, filesize=38.8 K 2024-11-10T19:17:54,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/3c05eb6f1f85486882dabea44f5f591a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/3c05eb6f1f85486882dabea44f5f591a 2024-11-10T19:17:54,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/3c05eb6f1f85486882dabea44f5f591a, entries=150, sequenceid=222, filesize=11.9 K 2024-11-10T19:17:54,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/0aaca40d3ece45d39d6fee1836130314 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/0aaca40d3ece45d39d6fee1836130314 2024-11-10T19:17:54,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/0aaca40d3ece45d39d6fee1836130314, entries=150, sequenceid=222, filesize=11.9 K 2024-11-10T19:17:54,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for bccf9fccd1002d9d7579e690ca8d056c in 655ms, sequenceid=222, compaction requested=false 2024-11-10T19:17:54,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:54,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:17:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-10T19:17:54,962 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:54,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105c76f044539c4535ae37f90fd03a8088_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266274282/Put/seqid=0 2024-11-10T19:17:54,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-10T19:17:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T19:17:54,970 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:54,971 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:54,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:54,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742223_1399 (size=14794) 2024-11-10T19:17:55,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266274940, max timestamp=1731266275042Num uniques 859; 2024-11-10T19:17:55,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:55,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266274940, max timestamp=1731266275042Num uniques 859; 2024-11-10T19:17:55,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:55,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266274940, max timestamp=1731266275042Num uniques 859; 2024-11-10T19:17:55,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T19:17:55,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:55,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266335133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266335150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266335153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,188 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5ec8c0ada5e246d1b9331c2d6500a64b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5ec8c0ada5e246d1b9331c2d6500a64b 2024-11-10T19:17:55,193 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/A of bccf9fccd1002d9d7579e690ca8d056c into 5ec8c0ada5e246d1b9331c2d6500a64b(size=30.7 K), total size for store is 69.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:55,193 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:55,193 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/A, priority=12, startTime=1731266274204; duration=0sec 2024-11-10T19:17:55,193 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:55,193 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:A 2024-11-10T19:17:55,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:55,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:55,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T19:17:55,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266335269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266335285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266335285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,379 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:55,382 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105c76f044539c4535ae37f90fd03a8088_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105c76f044539c4535ae37f90fd03a8088_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:55,383 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/301da1808ad94fad914a1a1bf24eafef, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:55,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/301da1808ad94fad914a1a1bf24eafef is 175, key is test_row_0/A:col10/1731266274282/Put/seqid=0 2024-11-10T19:17:55,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742224_1400 (size=39749) 2024-11-10T19:17:55,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/301da1808ad94fad914a1a1bf24eafef 2024-11-10T19:17:55,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/5fc9ded054b943e490c5db4d38896cd0 is 50, key is test_row_0/B:col10/1731266274282/Put/seqid=0 2024-11-10T19:17:55,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:55,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:55,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742225_1401 (size=12151) 2024-11-10T19:17:55,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/5fc9ded054b943e490c5db4d38896cd0 2024-11-10T19:17:55,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/3faad7b080cc4659854fd22aa728f1aa is 50, key is test_row_0/C:col10/1731266274282/Put/seqid=0 2024-11-10T19:17:55,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266335492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266335501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742226_1402 (size=12151) 2024-11-10T19:17:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266335504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:55,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:55,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T19:17:55,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:55,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:55,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266335811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266335822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266335831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:55,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:55,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:55,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:55,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/3faad7b080cc4659854fd22aa728f1aa 2024-11-10T19:17:55,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/301da1808ad94fad914a1a1bf24eafef as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/301da1808ad94fad914a1a1bf24eafef 2024-11-10T19:17:55,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/301da1808ad94fad914a1a1bf24eafef, entries=200, sequenceid=234, filesize=38.8 K 2024-11-10T19:17:55,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/5fc9ded054b943e490c5db4d38896cd0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5fc9ded054b943e490c5db4d38896cd0 2024-11-10T19:17:55,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5fc9ded054b943e490c5db4d38896cd0, entries=150, sequenceid=234, filesize=11.9 K 2024-11-10T19:17:55,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/3faad7b080cc4659854fd22aa728f1aa as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3faad7b080cc4659854fd22aa728f1aa 2024-11-10T19:17:55,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3faad7b080cc4659854fd22aa728f1aa, entries=150, sequenceid=234, filesize=11.9 K 2024-11-10T19:17:55,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bccf9fccd1002d9d7579e690ca8d056c in 988ms, sequenceid=234, compaction requested=true 2024-11-10T19:17:55,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:55,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:55,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:55,926 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:55,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:55,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:55,926 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:55,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:55,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:55,926 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:55,926 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:55,927 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/A is initiating minor compaction (all files) 2024-11-10T19:17:55,927 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/B is initiating minor compaction (all files) 2024-11-10T19:17:55,927 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/B in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,927 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/A in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,927 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5ec8c0ada5e246d1b9331c2d6500a64b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9e89d099ee5b4f699049ebf54b1ef9c7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/301da1808ad94fad914a1a1bf24eafef] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=108.4 K 2024-11-10T19:17:55,927 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/76588007bf6449d4ae212c3fb8939e35, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/3c05eb6f1f85486882dabea44f5f591a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5fc9ded054b943e490c5db4d38896cd0] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=36.0 K 2024-11-10T19:17:55,927 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,927 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5ec8c0ada5e246d1b9331c2d6500a64b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9e89d099ee5b4f699049ebf54b1ef9c7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/301da1808ad94fad914a1a1bf24eafef] 2024-11-10T19:17:55,927 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 76588007bf6449d4ae212c3fb8939e35, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731266272466 2024-11-10T19:17:55,927 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ec8c0ada5e246d1b9331c2d6500a64b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731266272466 2024-11-10T19:17:55,927 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e89d099ee5b4f699049ebf54b1ef9c7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731266273840 2024-11-10T19:17:55,928 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c05eb6f1f85486882dabea44f5f591a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731266273840 2024-11-10T19:17:55,928 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 301da1808ad94fad914a1a1bf24eafef, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731266274260 2024-11-10T19:17:55,928 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc9ded054b943e490c5db4d38896cd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731266274271 2024-11-10T19:17:55,935 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:55,937 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#B#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:55,938 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/cf6c33c973c2482fa3937486e419a37c is 50, key is test_row_0/B:col10/1731266274282/Put/seqid=0 2024-11-10T19:17:55,941 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111084245893bd8e48e491f6d1ad511edb26_bccf9fccd1002d9d7579e690ca8d056c store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:55,943 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111084245893bd8e48e491f6d1ad511edb26_bccf9fccd1002d9d7579e690ca8d056c, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:55,943 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111084245893bd8e48e491f6d1ad511edb26_bccf9fccd1002d9d7579e690ca8d056c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:55,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742227_1403 (size=12629) 2024-11-10T19:17:55,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742228_1404 (size=4469) 2024-11-10T19:17:55,959 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/cf6c33c973c2482fa3937486e419a37c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/cf6c33c973c2482fa3937486e419a37c 2024-11-10T19:17:55,963 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/B of bccf9fccd1002d9d7579e690ca8d056c into cf6c33c973c2482fa3937486e419a37c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:55,963 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:55,963 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/B, priority=13, startTime=1731266275926; duration=0sec 2024-11-10T19:17:55,963 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:55,963 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:B 2024-11-10T19:17:55,963 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:17:55,964 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:17:55,964 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/C is initiating minor compaction (all files) 2024-11-10T19:17:55,964 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/C in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:55,964 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/e2f60d1f936b4c5698cdd38246124808, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/0aaca40d3ece45d39d6fee1836130314, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3faad7b080cc4659854fd22aa728f1aa] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=36.0 K 2024-11-10T19:17:55,964 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f60d1f936b4c5698cdd38246124808, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731266272466 2024-11-10T19:17:55,964 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aaca40d3ece45d39d6fee1836130314, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731266273840 2024-11-10T19:17:55,965 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3faad7b080cc4659854fd22aa728f1aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731266274271 2024-11-10T19:17:55,971 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:55,971 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/72f7e3919a2547ecb1a2eded34be75bf is 50, key is test_row_0/C:col10/1731266274282/Put/seqid=0 2024-11-10T19:17:55,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742229_1405 (size=12629) 2024-11-10T19:17:55,997 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/72f7e3919a2547ecb1a2eded34be75bf as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/72f7e3919a2547ecb1a2eded34be75bf 2024-11-10T19:17:56,004 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/C of bccf9fccd1002d9d7579e690ca8d056c into 72f7e3919a2547ecb1a2eded34be75bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:56,004 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:56,004 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/C, priority=13, startTime=1731266275926; duration=0sec 2024-11-10T19:17:56,004 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:56,005 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:C 2024-11-10T19:17:56,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-10T19:17:56,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:56,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:56,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:56,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:56,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:56,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:17:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:56,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e1b024cff8034ef6a84e8c8402fc5de1_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266275123/Put/seqid=0 2024-11-10T19:17:56,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742230_1406 (size=14994) 2024-11-10T19:17:56,031 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:56,034 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e1b024cff8034ef6a84e8c8402fc5de1_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e1b024cff8034ef6a84e8c8402fc5de1_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:56,035 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/49f9b696ca2e441e99898cc3137a93e6, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:56,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/49f9b696ca2e441e99898cc3137a93e6 is 175, key is test_row_0/A:col10/1731266275123/Put/seqid=0 2024-11-10T19:17:56,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:56,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:56,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:56,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:56,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266336049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742231_1407 (size=39949) 2024-11-10T19:17:56,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266336050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T19:17:56,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266336164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266336171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:56,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:56,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:56,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:56,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266336321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:56,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,359 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#A#compaction#336 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:56,359 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/f886159e0d5f473d911a101c40cea0b0 is 175, key is test_row_0/A:col10/1731266274282/Put/seqid=0 2024-11-10T19:17:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266336353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742232_1408 (size=31583) 2024-11-10T19:17:56,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266336361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,380 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/f886159e0d5f473d911a101c40cea0b0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f886159e0d5f473d911a101c40cea0b0 2024-11-10T19:17:56,388 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/A of bccf9fccd1002d9d7579e690ca8d056c into f886159e0d5f473d911a101c40cea0b0(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:56,388 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:56,388 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/A, priority=13, startTime=1731266275926; duration=0sec 2024-11-10T19:17:56,388 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:56,388 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:A 2024-11-10T19:17:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266336392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266336393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,462 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/49f9b696ca2e441e99898cc3137a93e6 2024-11-10T19:17:56,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/5cf591a3c42c41ad83961eaf9f959d1c is 50, key is test_row_0/B:col10/1731266275123/Put/seqid=0 2024-11-10T19:17:56,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742233_1409 (size=12301) 2024-11-10T19:17:56,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/5cf591a3c42c41ad83961eaf9f959d1c 2024-11-10T19:17:56,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:56,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:56,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:56,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:56,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:56,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/51610d584e754b138d216ac129d7f2c4 is 50, key is test_row_0/C:col10/1731266275123/Put/seqid=0 2024-11-10T19:17:56,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742234_1410 (size=12301) 2024-11-10T19:17:56,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/51610d584e754b138d216ac129d7f2c4 2024-11-10T19:17:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/49f9b696ca2e441e99898cc3137a93e6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/49f9b696ca2e441e99898cc3137a93e6 2024-11-10T19:17:56,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/49f9b696ca2e441e99898cc3137a93e6, entries=200, sequenceid=274, filesize=39.0 K 2024-11-10T19:17:56,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/5cf591a3c42c41ad83961eaf9f959d1c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5cf591a3c42c41ad83961eaf9f959d1c 2024-11-10T19:17:56,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5cf591a3c42c41ad83961eaf9f959d1c, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T19:17:56,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/51610d584e754b138d216ac129d7f2c4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/51610d584e754b138d216ac129d7f2c4 2024-11-10T19:17:56,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/51610d584e754b138d216ac129d7f2c4, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T19:17:56,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for bccf9fccd1002d9d7579e690ca8d056c in 552ms, sequenceid=274, compaction requested=false 2024-11-10T19:17:56,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:56,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-10T19:17:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:56,661 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T19:17:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:56,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105fc3cd84d4044f1ebad4863cd72da45d_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266276048/Put/seqid=0 2024-11-10T19:17:56,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742235_1411 (size=12454) 2024-11-10T19:17:56,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:56,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266276768, max timestamp=1731266276794Num uniques 859; 2024-11-10T19:17:56,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:56,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266276768, max timestamp=1731266276794Num uniques 859; 2024-11-10T19:17:56,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:56,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266276768, max timestamp=1731266276794Num uniques 859; 2024-11-10T19:17:56,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:17:56,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266336919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266336921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266337041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266337051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:57,077 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105fc3cd84d4044f1ebad4863cd72da45d_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105fc3cd84d4044f1ebad4863cd72da45d_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:57,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/413a2fefa04449f7b400add315b5f30f, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:57,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/413a2fefa04449f7b400add315b5f30f is 175, key is test_row_0/A:col10/1731266276048/Put/seqid=0 2024-11-10T19:17:57,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742236_1412 (size=31255) 2024-11-10T19:17:57,092 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/413a2fefa04449f7b400add315b5f30f 2024-11-10T19:17:57,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/54f69552d4a74c44a04363419fe90ebc is 50, key is test_row_0/B:col10/1731266276048/Put/seqid=0 2024-11-10T19:17:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T19:17:57,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742237_1413 (size=12301) 2024-11-10T19:17:57,145 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/54f69552d4a74c44a04363419fe90ebc 2024-11-10T19:17:57,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/eafb5d25d1a148d4be0807025bf90334 is 50, key is test_row_0/C:col10/1731266276048/Put/seqid=0 2024-11-10T19:17:57,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742238_1414 (size=12301) 2024-11-10T19:17:57,190 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/eafb5d25d1a148d4be0807025bf90334 2024-11-10T19:17:57,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/413a2fefa04449f7b400add315b5f30f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/413a2fefa04449f7b400add315b5f30f 2024-11-10T19:17:57,199 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/413a2fefa04449f7b400add315b5f30f, entries=150, sequenceid=284, filesize=30.5 K 2024-11-10T19:17:57,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/54f69552d4a74c44a04363419fe90ebc as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/54f69552d4a74c44a04363419fe90ebc 2024-11-10T19:17:57,203 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/54f69552d4a74c44a04363419fe90ebc, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T19:17:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/eafb5d25d1a148d4be0807025bf90334 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/eafb5d25d1a148d4be0807025bf90334 2024-11-10T19:17:57,209 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/eafb5d25d1a148d4be0807025bf90334, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T19:17:57,210 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for bccf9fccd1002d9d7579e690ca8d056c in 550ms, sequenceid=284, compaction requested=true 2024-11-10T19:17:57,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:57,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:57,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-10T19:17:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-10T19:17:57,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-10T19:17:57,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2400 sec 2024-11-10T19:17:57,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.2510 sec 2024-11-10T19:17:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:57,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-10T19:17:57,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:57,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T19:17:57,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:57,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T19:17:57,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:57,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T19:17:57,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bfd0d2cb29544f71b5f86a86da75b514_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266276866/Put/seqid=0 2024-11-10T19:17:57,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742239_1415 (size=14994) 2024-11-10T19:17:57,339 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:57,350 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bfd0d2cb29544f71b5f86a86da75b514_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bfd0d2cb29544f71b5f86a86da75b514_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:57,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266337326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266337326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,358 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/a27eb07fa7694b7f9580bdc8c11272f3, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:57,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/a27eb07fa7694b7f9580bdc8c11272f3 is 175, key is test_row_0/A:col10/1731266276866/Put/seqid=0 2024-11-10T19:17:57,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266337344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,366 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742240_1416 (size=39949) 2024-11-10T19:17:57,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266337382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,401 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266337383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,403 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266337462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266337462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266337684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266337696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:57,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/a27eb07fa7694b7f9580bdc8c11272f3 2024-11-10T19:17:57,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/c837cef185b440d886d96d2da4dec868 is 50, key is test_row_0/B:col10/1731266276866/Put/seqid=0 2024-11-10T19:17:57,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742241_1417 (size=12301) 2024-11-10T19:17:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266338011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:58,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266338024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:58,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/c837cef185b440d886d96d2da4dec868 2024-11-10T19:17:58,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/360e9b92f88d4b0d89d342680d3a7901 is 50, key is test_row_0/C:col10/1731266276866/Put/seqid=0 2024-11-10T19:17:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742242_1418 (size=12301) 2024-11-10T19:17:58,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/360e9b92f88d4b0d89d342680d3a7901 2024-11-10T19:17:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/a27eb07fa7694b7f9580bdc8c11272f3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/a27eb07fa7694b7f9580bdc8c11272f3 2024-11-10T19:17:58,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/a27eb07fa7694b7f9580bdc8c11272f3, entries=200, sequenceid=322, filesize=39.0 K 2024-11-10T19:17:58,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/c837cef185b440d886d96d2da4dec868 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c837cef185b440d886d96d2da4dec868 2024-11-10T19:17:58,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c837cef185b440d886d96d2da4dec868, entries=150, sequenceid=322, filesize=12.0 K 2024-11-10T19:17:58,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/360e9b92f88d4b0d89d342680d3a7901 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/360e9b92f88d4b0d89d342680d3a7901 2024-11-10T19:17:58,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/360e9b92f88d4b0d89d342680d3a7901, entries=150, sequenceid=322, filesize=12.0 K 2024-11-10T19:17:58,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for bccf9fccd1002d9d7579e690ca8d056c in 1269ms, sequenceid=322, compaction requested=true 2024-11-10T19:17:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:58,548 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:58,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:17:58,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:58,551 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:58,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:17:58,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:58,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:17:58,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:58,555 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:58,555 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/B is initiating minor compaction (all files) 2024-11-10T19:17:58,556 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/B in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:58,556 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/cf6c33c973c2482fa3937486e419a37c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5cf591a3c42c41ad83961eaf9f959d1c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/54f69552d4a74c44a04363419fe90ebc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c837cef185b440d886d96d2da4dec868] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=48.4 K 2024-11-10T19:17:58,559 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142736 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:58,559 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/A is initiating minor compaction (all files) 2024-11-10T19:17:58,559 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/A in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:58,559 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting cf6c33c973c2482fa3937486e419a37c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731266274271 2024-11-10T19:17:58,559 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f886159e0d5f473d911a101c40cea0b0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/49f9b696ca2e441e99898cc3137a93e6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/413a2fefa04449f7b400add315b5f30f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/a27eb07fa7694b7f9580bdc8c11272f3] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=139.4 K 2024-11-10T19:17:58,560 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:58,560 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f886159e0d5f473d911a101c40cea0b0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/49f9b696ca2e441e99898cc3137a93e6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/413a2fefa04449f7b400add315b5f30f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/a27eb07fa7694b7f9580bdc8c11272f3] 2024-11-10T19:17:58,560 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf591a3c42c41ad83961eaf9f959d1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731266275100 2024-11-10T19:17:58,560 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f886159e0d5f473d911a101c40cea0b0, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731266274271 2024-11-10T19:17:58,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:17:58,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:17:58,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:58,564 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49f9b696ca2e441e99898cc3137a93e6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731266275100 2024-11-10T19:17:58,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:17:58,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:58,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:17:58,565 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f69552d4a74c44a04363419fe90ebc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731266276028 2024-11-10T19:17:58,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:17:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:58,565 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c837cef185b440d886d96d2da4dec868, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731266276866 2024-11-10T19:17:58,565 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 413a2fefa04449f7b400add315b5f30f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731266276028 2024-11-10T19:17:58,566 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting a27eb07fa7694b7f9580bdc8c11272f3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731266276866 2024-11-10T19:17:58,593 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:58,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e7ab8bb5c73e43bf9888a9b68bd1a763_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266277324/Put/seqid=0 2024-11-10T19:17:58,603 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#B#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:58,604 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/30117294eafa4e6799089014a020f4ba is 50, key is test_row_0/B:col10/1731266276866/Put/seqid=0 2024-11-10T19:17:58,607 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110a4d8d4db5dd44214badc9215485d2770_bccf9fccd1002d9d7579e690ca8d056c store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:58,610 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110a4d8d4db5dd44214badc9215485d2770_bccf9fccd1002d9d7579e690ca8d056c, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:58,610 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a4d8d4db5dd44214badc9215485d2770_bccf9fccd1002d9d7579e690ca8d056c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:58,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742244_1420 (size=12915) 2024-11-10T19:17:58,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742243_1419 (size=14994) 2024-11-10T19:17:58,701 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/30117294eafa4e6799089014a020f4ba as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/30117294eafa4e6799089014a020f4ba 2024-11-10T19:17:58,706 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/B of bccf9fccd1002d9d7579e690ca8d056c into 30117294eafa4e6799089014a020f4ba(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:58,706 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:58,707 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/B, priority=12, startTime=1731266278551; duration=0sec 2024-11-10T19:17:58,707 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:17:58,707 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:B 2024-11-10T19:17:58,707 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:17:58,708 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:17:58,708 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/C is initiating minor compaction (all files) 2024-11-10T19:17:58,708 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/C in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:58,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742245_1421 (size=4469) 2024-11-10T19:17:58,708 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/72f7e3919a2547ecb1a2eded34be75bf, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/51610d584e754b138d216ac129d7f2c4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/eafb5d25d1a148d4be0807025bf90334, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/360e9b92f88d4b0d89d342680d3a7901] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=48.4 K 2024-11-10T19:17:58,708 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 72f7e3919a2547ecb1a2eded34be75bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731266274271 2024-11-10T19:17:58,709 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 51610d584e754b138d216ac129d7f2c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731266275100 2024-11-10T19:17:58,709 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting eafb5d25d1a148d4be0807025bf90334, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731266276028 2024-11-10T19:17:58,709 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 360e9b92f88d4b0d89d342680d3a7901, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731266276866 2024-11-10T19:17:58,722 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#C#compaction#351 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:58,723 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4bbba0987d394c12ac1c89c6ef2dfe28 is 50, key is test_row_0/C:col10/1731266276866/Put/seqid=0 2024-11-10T19:17:58,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742246_1422 (size=12915) 2024-11-10T19:17:58,795 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4bbba0987d394c12ac1c89c6ef2dfe28 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4bbba0987d394c12ac1c89c6ef2dfe28 2024-11-10T19:17:58,803 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/C of bccf9fccd1002d9d7579e690ca8d056c into 4bbba0987d394c12ac1c89c6ef2dfe28(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:58,803 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:58,803 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/C, priority=12, startTime=1731266278551; duration=0sec 2024-11-10T19:17:58,804 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:58,804 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:C 2024-11-10T19:17:58,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266278614, max timestamp=1731266278809Num uniques 859; 2024-11-10T19:17:58,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:17:58,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266338867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:58,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266338898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:58,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:58,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:58,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266339004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266339032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:59,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,093 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:17:59,097 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e7ab8bb5c73e43bf9888a9b68bd1a763_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e7ab8bb5c73e43bf9888a9b68bd1a763_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:17:59,098 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5adea20be9744d7dac6fb432cf9eeeac, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:17:59,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5adea20be9744d7dac6fb432cf9eeeac is 175, key is test_row_0/A:col10/1731266277324/Put/seqid=0 2024-11-10T19:17:59,113 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#A#compaction#348 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:17:59,113 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/9ad7828c50014ecbafbe2bcb7d8923f7 is 175, key is test_row_0/A:col10/1731266276866/Put/seqid=0 2024-11-10T19:17:59,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742247_1423 (size=39949) 2024-11-10T19:17:59,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742248_1424 (size=31869) 2024-11-10T19:17:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-10T19:17:59,132 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:17:59,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:17:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-10T19:17:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T19:17:59,134 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:17:59,134 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:17:59,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:17:59,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266339224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T19:17:59,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266339251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T19:17:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266339381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:59,396 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266339411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:59,417 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266339422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:59,440 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4290 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:17:59,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T19:17:59,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T19:17:59,521 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5adea20be9744d7dac6fb432cf9eeeac 2024-11-10T19:17:59,532 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/9ad7828c50014ecbafbe2bcb7d8923f7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9ad7828c50014ecbafbe2bcb7d8923f7 2024-11-10T19:17:59,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/4d65c68072764599b3d0265485f53c87 is 50, key is test_row_0/B:col10/1731266277324/Put/seqid=0 2024-11-10T19:17:59,539 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/A of bccf9fccd1002d9d7579e690ca8d056c into 9ad7828c50014ecbafbe2bcb7d8923f7(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:17:59,539 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:17:59,539 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/A, priority=12, startTime=1731266278548; duration=0sec 2024-11-10T19:17:59,539 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:17:59,539 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:A 2024-11-10T19:17:59,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742249_1425 (size=12301) 2024-11-10T19:17:59,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/4d65c68072764599b3d0265485f53c87 2024-11-10T19:17:59,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/39bf37c7b7a940c7ac6a1657667dc7c1 is 50, key is test_row_0/C:col10/1731266277324/Put/seqid=0 2024-11-10T19:17:59,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742250_1426 (size=12301) 2024-11-10T19:17:59,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266339553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:17:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266339571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:17:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:17:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:17:59,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T19:17:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T19:17:59,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:59,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T19:17:59,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T19:17:59,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:17:59,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:17:59,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:17:59,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/39bf37c7b7a940c7ac6a1657667dc7c1 2024-11-10T19:17:59,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5adea20be9744d7dac6fb432cf9eeeac as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5adea20be9744d7dac6fb432cf9eeeac 2024-11-10T19:17:59,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5adea20be9744d7dac6fb432cf9eeeac, entries=200, sequenceid=332, filesize=39.0 K 2024-11-10T19:17:59,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/4d65c68072764599b3d0265485f53c87 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/4d65c68072764599b3d0265485f53c87 2024-11-10T19:17:59,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/4d65c68072764599b3d0265485f53c87, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T19:17:59,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/39bf37c7b7a940c7ac6a1657667dc7c1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/39bf37c7b7a940c7ac6a1657667dc7c1 2024-11-10T19:17:59,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/39bf37c7b7a940c7ac6a1657667dc7c1, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T19:17:59,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for bccf9fccd1002d9d7579e690ca8d056c in 1424ms, sequenceid=332, compaction requested=false 2024-11-10T19:17:59,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:00,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T19:18:00,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:00,066 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=129.71 KB heapSize=381.64 KB 2024-11-10T19:18:00,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:18:00,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:00,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:18:00,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:00,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:18:00,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:00,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107c62046651f74985bc5c5d88e709c361_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266278837/Put/seqid=0 2024-11-10T19:18:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:00,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:18:00,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742251_1427 (size=12454) 2024-11-10T19:18:00,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:00,090 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107c62046651f74985bc5c5d88e709c361_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107c62046651f74985bc5c5d88e709c361_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:00,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5c5bb1e2f5cd4343a4fd5535d45e761b, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:00,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5c5bb1e2f5cd4343a4fd5535d45e761b is 175, key is test_row_0/A:col10/1731266278837/Put/seqid=0 2024-11-10T19:18:00,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742252_1428 (size=31255) 2024-11-10T19:18:00,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266340150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266340153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:00,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266340261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T19:18:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266340271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266340471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:00,503 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5c5bb1e2f5cd4343a4fd5535d45e761b 2024-11-10T19:18:00,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266340491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:00,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/da2aa2e8a1894f46aed6950d724307ea is 50, key is test_row_0/B:col10/1731266278837/Put/seqid=0 2024-11-10T19:18:00,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742253_1429 (size=12301) 2024-11-10T19:18:00,529 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/da2aa2e8a1894f46aed6950d724307ea 2024-11-10T19:18:00,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/c95d66a1005c4ff2a389702096fb1477 is 50, key is test_row_0/C:col10/1731266278837/Put/seqid=0 2024-11-10T19:18:00,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742254_1430 (size=12301) 2024-11-10T19:18:00,561 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/c95d66a1005c4ff2a389702096fb1477 2024-11-10T19:18:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/5c5bb1e2f5cd4343a4fd5535d45e761b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5c5bb1e2f5cd4343a4fd5535d45e761b 2024-11-10T19:18:00,573 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5c5bb1e2f5cd4343a4fd5535d45e761b, entries=150, sequenceid=362, filesize=30.5 K 2024-11-10T19:18:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/da2aa2e8a1894f46aed6950d724307ea as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/da2aa2e8a1894f46aed6950d724307ea 2024-11-10T19:18:00,576 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/da2aa2e8a1894f46aed6950d724307ea, entries=150, sequenceid=362, filesize=12.0 K 2024-11-10T19:18:00,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/c95d66a1005c4ff2a389702096fb1477 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c95d66a1005c4ff2a389702096fb1477 2024-11-10T19:18:00,582 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c95d66a1005c4ff2a389702096fb1477, entries=150, sequenceid=362, filesize=12.0 K 2024-11-10T19:18:00,589 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~381.59 KB/390752, currentSize=46.96 KB/48090 for bccf9fccd1002d9d7579e690ca8d056c in 523ms, sequenceid=362, compaction requested=true 2024-11-10T19:18:00,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:00,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:00,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-10T19:18:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-10T19:18:00,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-10T19:18:00,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4560 sec 2024-11-10T19:18:00,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.4610 sec 2024-11-10T19:18:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:00,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T19:18:00,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:18:00,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:00,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:18:00,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:00,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:18:00,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:00,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110695faef98d7b421ca495fa1a8a86f7c2_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266280138/Put/seqid=0 2024-11-10T19:18:00,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742255_1431 (size=14994) 2024-11-10T19:18:00,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266280813, max timestamp=1731266280922Num uniques 859; 2024-11-10T19:18:00,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:18:00,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266280813, max timestamp=1731266280922Num uniques 859; 2024-11-10T19:18:00,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:18:00,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731266280813, max timestamp=1731266280922Num uniques 859; 2024-11-10T19:18:00,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:18:01,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266341026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:01,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266341048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:01,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266341151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:01,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266341171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:01,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:01,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110695faef98d7b421ca495fa1a8a86f7c2_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110695faef98d7b421ca495fa1a8a86f7c2_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:01,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/2f4a165fb42b4d05ae1fa3157dcc59c7, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:01,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/2f4a165fb42b4d05ae1fa3157dcc59c7 is 175, key is test_row_0/A:col10/1731266280138/Put/seqid=0 2024-11-10T19:18:01,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742256_1432 (size=39949) 2024-11-10T19:18:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T19:18:01,283 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:01,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-10T19:18:01,285 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T19:18:01,286 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:01,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:01,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266341372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T19:18:01,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266341391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:01,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T19:18:01,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:18:01,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:01,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:01,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T19:18:01,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:18:01,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:01,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T19:18:01,670 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/2f4a165fb42b4d05ae1fa3157dcc59c7 2024-11-10T19:18:01,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/c1fc0de4bd474421af3719dd3bf4c9e2 is 50, key is test_row_0/B:col10/1731266280138/Put/seqid=0 2024-11-10T19:18:01,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742257_1433 (size=12301) 2024-11-10T19:18:01,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/c1fc0de4bd474421af3719dd3bf4c9e2 2024-11-10T19:18:01,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266341691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:01,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:01,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/849f7d4b3aa34d758bffa8e53a12d40b is 50, key is test_row_0/C:col10/1731266280138/Put/seqid=0 2024-11-10T19:18:01,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266341711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:01,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742258_1434 (size=12301) 2024-11-10T19:18:01,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/849f7d4b3aa34d758bffa8e53a12d40b 2024-11-10T19:18:01,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T19:18:01,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:18:01,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:01,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:01,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/2f4a165fb42b4d05ae1fa3157dcc59c7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2f4a165fb42b4d05ae1fa3157dcc59c7 2024-11-10T19:18:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:01,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2f4a165fb42b4d05ae1fa3157dcc59c7, entries=200, sequenceid=373, filesize=39.0 K 2024-11-10T19:18:01,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/c1fc0de4bd474421af3719dd3bf4c9e2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c1fc0de4bd474421af3719dd3bf4c9e2 2024-11-10T19:18:01,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c1fc0de4bd474421af3719dd3bf4c9e2, entries=150, sequenceid=373, filesize=12.0 K 2024-11-10T19:18:01,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/849f7d4b3aa34d758bffa8e53a12d40b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/849f7d4b3aa34d758bffa8e53a12d40b 2024-11-10T19:18:01,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/849f7d4b3aa34d758bffa8e53a12d40b, entries=150, sequenceid=373, filesize=12.0 K 2024-11-10T19:18:01,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bccf9fccd1002d9d7579e690ca8d056c in 986ms, sequenceid=373, compaction requested=true 2024-11-10T19:18:01,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:01,784 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:01,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:01,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:01,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:01,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:01,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:01,784 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:01,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:01,785 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:01,786 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/A is initiating minor compaction (all files) 2024-11-10T19:18:01,786 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/A in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,786 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9ad7828c50014ecbafbe2bcb7d8923f7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5adea20be9744d7dac6fb432cf9eeeac, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5c5bb1e2f5cd4343a4fd5535d45e761b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2f4a165fb42b4d05ae1fa3157dcc59c7] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=139.7 K 2024-11-10T19:18:01,786 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,786 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9ad7828c50014ecbafbe2bcb7d8923f7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5adea20be9744d7dac6fb432cf9eeeac, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5c5bb1e2f5cd4343a4fd5535d45e761b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2f4a165fb42b4d05ae1fa3157dcc59c7] 2024-11-10T19:18:01,786 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:01,786 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/B is initiating minor compaction (all files) 2024-11-10T19:18:01,786 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad7828c50014ecbafbe2bcb7d8923f7, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731266276866 2024-11-10T19:18:01,786 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/B in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,786 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/30117294eafa4e6799089014a020f4ba, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/4d65c68072764599b3d0265485f53c87, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/da2aa2e8a1894f46aed6950d724307ea, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c1fc0de4bd474421af3719dd3bf4c9e2] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=48.7 K 2024-11-10T19:18:01,786 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5adea20be9744d7dac6fb432cf9eeeac, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731266277295 2024-11-10T19:18:01,787 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 30117294eafa4e6799089014a020f4ba, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731266276866 2024-11-10T19:18:01,787 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c5bb1e2f5cd4343a4fd5535d45e761b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731266278837 2024-11-10T19:18:01,787 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d65c68072764599b3d0265485f53c87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731266277295 2024-11-10T19:18:01,788 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f4a165fb42b4d05ae1fa3157dcc59c7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731266280138 2024-11-10T19:18:01,788 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting da2aa2e8a1894f46aed6950d724307ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731266278837 2024-11-10T19:18:01,788 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c1fc0de4bd474421af3719dd3bf4c9e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731266280138 2024-11-10T19:18:01,795 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:01,798 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#B#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:01,798 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/82bc0ca44bec4064b560dc0555d554c8 is 50, key is test_row_0/B:col10/1731266280138/Put/seqid=0 2024-11-10T19:18:01,801 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100c8eecd87f854d94aa25cda109431947_bccf9fccd1002d9d7579e690ca8d056c store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:01,803 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100c8eecd87f854d94aa25cda109431947_bccf9fccd1002d9d7579e690ca8d056c, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:01,804 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100c8eecd87f854d94aa25cda109431947_bccf9fccd1002d9d7579e690ca8d056c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:01,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742259_1435 (size=13051) 2024-11-10T19:18:01,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742260_1436 (size=4469) 2024-11-10T19:18:01,828 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#A#compaction#361 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:01,829 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/1c7acf38aab54447b19ae62b64d33726 is 175, key is test_row_0/A:col10/1731266280138/Put/seqid=0 2024-11-10T19:18:01,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742261_1437 (size=32005) 2024-11-10T19:18:01,843 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/1c7acf38aab54447b19ae62b64d33726 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1c7acf38aab54447b19ae62b64d33726 2024-11-10T19:18:01,848 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/A of bccf9fccd1002d9d7579e690ca8d056c into 1c7acf38aab54447b19ae62b64d33726(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:01,848 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:01,848 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/A, priority=12, startTime=1731266281784; duration=0sec 2024-11-10T19:18:01,848 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:01,848 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:A 2024-11-10T19:18:01,848 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:01,850 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:01,850 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/C is initiating minor compaction (all files) 2024-11-10T19:18:01,850 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/C in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,850 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4bbba0987d394c12ac1c89c6ef2dfe28, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/39bf37c7b7a940c7ac6a1657667dc7c1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c95d66a1005c4ff2a389702096fb1477, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/849f7d4b3aa34d758bffa8e53a12d40b] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=48.7 K 2024-11-10T19:18:01,850 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bbba0987d394c12ac1c89c6ef2dfe28, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731266276866 2024-11-10T19:18:01,851 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39bf37c7b7a940c7ac6a1657667dc7c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731266277295 2024-11-10T19:18:01,851 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c95d66a1005c4ff2a389702096fb1477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731266278837 2024-11-10T19:18:01,852 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 849f7d4b3aa34d758bffa8e53a12d40b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731266280138 2024-11-10T19:18:01,870 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:01,870 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4b751698b33443978a9be107d86bd56e is 50, key is test_row_0/C:col10/1731266280138/Put/seqid=0 2024-11-10T19:18:01,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742262_1438 (size=13051) 2024-11-10T19:18:01,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T19:18:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:01,909 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T19:18:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:18:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:18:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:18:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T19:18:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b2143be700dc416da9f0564c72322e17_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266281001/Put/seqid=0 2024-11-10T19:18:01,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742263_1439 (size=12454) 2024-11-10T19:18:01,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:01,959 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b2143be700dc416da9f0564c72322e17_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b2143be700dc416da9f0564c72322e17_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/442a8fd1704349e8965f785ea24a77cb, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/442a8fd1704349e8965f785ea24a77cb is 175, key is test_row_0/A:col10/1731266281001/Put/seqid=0 2024-11-10T19:18:01,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742264_1440 (size=31255) 2024-11-10T19:18:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:02,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:18:02,247 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/82bc0ca44bec4064b560dc0555d554c8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/82bc0ca44bec4064b560dc0555d554c8 2024-11-10T19:18:02,258 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/B of bccf9fccd1002d9d7579e690ca8d056c into 82bc0ca44bec4064b560dc0555d554c8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:02,258 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:02,258 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/B, priority=12, startTime=1731266281784; duration=0sec 2024-11-10T19:18:02,258 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:02,258 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:B 2024-11-10T19:18:02,284 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4b751698b33443978a9be107d86bd56e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4b751698b33443978a9be107d86bd56e 2024-11-10T19:18:02,302 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/C of bccf9fccd1002d9d7579e690ca8d056c into 4b751698b33443978a9be107d86bd56e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:02,302 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:02,302 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/C, priority=12, startTime=1731266281784; duration=0sec 2024-11-10T19:18:02,302 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:02,302 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:C 2024-11-10T19:18:02,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266342300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:02,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266342300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:02,371 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/442a8fd1704349e8965f785ea24a77cb 2024-11-10T19:18:02,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/68542c9b48844811a39f97d4551f318f is 50, key is test_row_0/B:col10/1731266281001/Put/seqid=0 2024-11-10T19:18:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742265_1441 (size=12301) 2024-11-10T19:18:02,410 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/68542c9b48844811a39f97d4551f318f 2024-11-10T19:18:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T19:18:02,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4c5958ab089044599b34e2b530cdc6ed is 50, key is test_row_0/C:col10/1731266281001/Put/seqid=0 2024-11-10T19:18:02,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266342422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:02,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266342432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:02,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:02,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742266_1442 (size=12301) 2024-11-10T19:18:02,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266342641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:02,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266342641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:02,840 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4c5958ab089044599b34e2b530cdc6ed 2024-11-10T19:18:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/442a8fd1704349e8965f785ea24a77cb as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/442a8fd1704349e8965f785ea24a77cb 2024-11-10T19:18:02,846 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/442a8fd1704349e8965f785ea24a77cb, entries=150, sequenceid=409, filesize=30.5 K 2024-11-10T19:18:02,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/68542c9b48844811a39f97d4551f318f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/68542c9b48844811a39f97d4551f318f 2024-11-10T19:18:02,850 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/68542c9b48844811a39f97d4551f318f, entries=150, sequenceid=409, filesize=12.0 K 2024-11-10T19:18:02,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/4c5958ab089044599b34e2b530cdc6ed as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4c5958ab089044599b34e2b530cdc6ed 2024-11-10T19:18:02,854 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4c5958ab089044599b34e2b530cdc6ed, entries=150, sequenceid=409, filesize=12.0 K 2024-11-10T19:18:02,854 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for bccf9fccd1002d9d7579e690ca8d056c in 945ms, sequenceid=409, compaction requested=false 2024-11-10T19:18:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-10T19:18:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-10T19:18:02,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-10T19:18:02,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-10T19:18:02,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5740 sec 2024-11-10T19:18:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:02,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T19:18:02,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:18:02,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:02,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:18:02,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:02,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:18:02,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:02,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f5e1f3a8b2b24b3d9f48ebac7a1c4874_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266282964/Put/seqid=0 2024-11-10T19:18:02,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742267_1443 (size=14994) 2024-11-10T19:18:03,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266343035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266343038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266343151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266343151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266343363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266343363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,384 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:03,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f5e1f3a8b2b24b3d9f48ebac7a1c4874_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f5e1f3a8b2b24b3d9f48ebac7a1c4874_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:03,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/1bf9505202d6441d9ad4a093bd701642, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:03,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/1bf9505202d6441d9ad4a093bd701642 is 175, key is test_row_0/A:col10/1731266282964/Put/seqid=0 2024-11-10T19:18:03,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742268_1444 (size=39949) 2024-11-10T19:18:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T19:18:03,432 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:03,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-10T19:18:03,434 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T19:18:03,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46136 deadline: 1731266343432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,435 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:03,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:03,435 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8302 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46148 deadline: 1731266343451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,461 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8308 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46158 deadline: 1731266343471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,484 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in bccf9fccd1002d9d7579e690ca8d056c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8334 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T19:18:03,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T19:18:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:03,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:18:03,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:03,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:03,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:03,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266343681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266343681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:03,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T19:18:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:18:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:03,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T19:18:03,805 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/1bf9505202d6441d9ad4a093bd701642 2024-11-10T19:18:03,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/bd2807376f6b44bea665c8feed259dcb is 50, key is test_row_0/B:col10/1731266282964/Put/seqid=0 2024-11-10T19:18:03,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742269_1445 (size=12301) 2024-11-10T19:18:03,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/bd2807376f6b44bea665c8feed259dcb 2024-11-10T19:18:03,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T19:18:03,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/2100322704e243e4a3cb1b2b380467f9 is 50, key is test_row_0/C:col10/1731266282964/Put/seqid=0 2024-11-10T19:18:03,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:03,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:18:03,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:03,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:03,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:03,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742270_1446 (size=12301) 2024-11-10T19:18:03,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/2100322704e243e4a3cb1b2b380467f9 2024-11-10T19:18:03,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/1bf9505202d6441d9ad4a093bd701642 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1bf9505202d6441d9ad4a093bd701642 2024-11-10T19:18:03,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1bf9505202d6441d9ad4a093bd701642, entries=200, sequenceid=423, filesize=39.0 K 2024-11-10T19:18:03,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/bd2807376f6b44bea665c8feed259dcb as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/bd2807376f6b44bea665c8feed259dcb 2024-11-10T19:18:03,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/bd2807376f6b44bea665c8feed259dcb, entries=150, sequenceid=423, filesize=12.0 K 2024-11-10T19:18:03,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/2100322704e243e4a3cb1b2b380467f9 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/2100322704e243e4a3cb1b2b380467f9 2024-11-10T19:18:03,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/2100322704e243e4a3cb1b2b380467f9, entries=150, sequenceid=423, filesize=12.0 K 2024-11-10T19:18:03,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bccf9fccd1002d9d7579e690ca8d056c in 1004ms, sequenceid=423, compaction requested=true 2024-11-10T19:18:03,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:03,969 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:03,970 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:03,970 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/A is initiating minor compaction (all files) 2024-11-10T19:18:03,970 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/A in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:03,970 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1c7acf38aab54447b19ae62b64d33726, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/442a8fd1704349e8965f785ea24a77cb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1bf9505202d6441d9ad4a093bd701642] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=100.8 K 2024-11-10T19:18:03,970 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:03,970 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1c7acf38aab54447b19ae62b64d33726, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/442a8fd1704349e8965f785ea24a77cb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1bf9505202d6441d9ad4a093bd701642] 2024-11-10T19:18:03,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:03,971 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c7acf38aab54447b19ae62b64d33726, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731266280138 2024-11-10T19:18:03,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:03,971 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:03,971 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 442a8fd1704349e8965f785ea24a77cb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731266281001 2024-11-10T19:18:03,972 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bf9505202d6441d9ad4a093bd701642, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731266282222 2024-11-10T19:18:03,972 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:03,972 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/B is initiating minor compaction (all files) 2024-11-10T19:18:03,972 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/B in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:03,972 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/82bc0ca44bec4064b560dc0555d554c8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/68542c9b48844811a39f97d4551f318f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/bd2807376f6b44bea665c8feed259dcb] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=36.8 K 2024-11-10T19:18:03,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:03,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:03,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:03,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:03,975 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 82bc0ca44bec4064b560dc0555d554c8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731266280138 2024-11-10T19:18:03,978 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 68542c9b48844811a39f97d4551f318f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731266281001 2024-11-10T19:18:03,979 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2807376f6b44bea665c8feed259dcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731266282222 2024-11-10T19:18:03,997 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:04,007 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110e11931f8aba6405988878e3b26a07226_bccf9fccd1002d9d7579e690ca8d056c store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:04,008 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:04,008 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/c4c386c8b3a5499f984698003353de69 is 50, key is test_row_0/B:col10/1731266282964/Put/seqid=0 2024-11-10T19:18:04,009 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110e11931f8aba6405988878e3b26a07226_bccf9fccd1002d9d7579e690ca8d056c, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:04,010 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e11931f8aba6405988878e3b26a07226_bccf9fccd1002d9d7579e690ca8d056c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:04,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742271_1447 (size=13153) 2024-11-10T19:18:04,025 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/c4c386c8b3a5499f984698003353de69 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c4c386c8b3a5499f984698003353de69 2024-11-10T19:18:04,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742272_1448 (size=4469) 2024-11-10T19:18:04,030 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/B of bccf9fccd1002d9d7579e690ca8d056c into c4c386c8b3a5499f984698003353de69(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:04,030 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:04,030 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/B, priority=13, startTime=1731266283971; duration=0sec 2024-11-10T19:18:04,030 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:04,030 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:B 2024-11-10T19:18:04,030 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:04,031 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:04,031 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/C is initiating minor compaction (all files) 2024-11-10T19:18:04,031 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/C in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:04,032 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4b751698b33443978a9be107d86bd56e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4c5958ab089044599b34e2b530cdc6ed, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/2100322704e243e4a3cb1b2b380467f9] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=36.8 K 2024-11-10T19:18:04,032 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b751698b33443978a9be107d86bd56e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731266280138 2024-11-10T19:18:04,032 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c5958ab089044599b34e2b530cdc6ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731266281001 2024-11-10T19:18:04,032 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 2100322704e243e4a3cb1b2b380467f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731266282222 2024-11-10T19:18:04,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T19:18:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:04,061 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T19:18:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T19:18:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:18:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:18:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:18:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:04,065 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#C#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:04,066 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/18ff425009144d55886c929485232e84 is 50, key is test_row_0/C:col10/1731266282964/Put/seqid=0 2024-11-10T19:18:04,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742273_1449 (size=13153) 2024-11-10T19:18:04,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108635f3c27f564190ae43c713d7692e02_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266283036/Put/seqid=0 2024-11-10T19:18:04,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742274_1450 (size=12454) 2024-11-10T19:18:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:04,097 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108635f3c27f564190ae43c713d7692e02_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108635f3c27f564190ae43c713d7692e02_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:04,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/654623a793164abf9b341dffd8cf115b, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:04,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/654623a793164abf9b341dffd8cf115b is 175, key is test_row_0/A:col10/1731266283036/Put/seqid=0 2024-11-10T19:18:04,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742275_1451 (size=31255) 2024-11-10T19:18:04,133 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=444, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/654623a793164abf9b341dffd8cf115b 2024-11-10T19:18:04,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/ad87cd0ebd57420fb6b99a5cb518a012 is 50, key is test_row_0/B:col10/1731266283036/Put/seqid=0 2024-11-10T19:18:04,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742276_1452 (size=12301) 2024-11-10T19:18:04,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. as already flushing 2024-11-10T19:18:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:04,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266344253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:04,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266344276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:04,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266344381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:04,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:04,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266344402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:04,431 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#A#compaction#369 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:04,432 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/0a3e64b0b545413ba504a5e5f8a52e04 is 175, key is test_row_0/A:col10/1731266282964/Put/seqid=0 2024-11-10T19:18:04,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742277_1453 (size=32107) 2024-11-10T19:18:04,470 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/0a3e64b0b545413ba504a5e5f8a52e04 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/0a3e64b0b545413ba504a5e5f8a52e04 2024-11-10T19:18:04,477 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/A of bccf9fccd1002d9d7579e690ca8d056c into 0a3e64b0b545413ba504a5e5f8a52e04(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:04,477 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:04,477 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/A, priority=13, startTime=1731266283969; duration=0sec 2024-11-10T19:18:04,477 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:04,477 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:A 2024-11-10T19:18:04,481 INFO [Thread-1591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1591. 2024-11-10T19:18:04,481 DEBUG [Thread-1591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:04,481 DEBUG [Thread-1591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,481 DEBUG [Thread-1591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:04,483 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-10T19:18:04,484 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:04,484 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,484 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:04,490 INFO [Thread-1589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1589. 2024-11-10T19:18:04,490 DEBUG [Thread-1589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:04,490 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,490 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,490 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:04,495 INFO [Thread-1587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1587. 2024-11-10T19:18:04,495 DEBUG [Thread-1587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:04,495 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,495 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:04,496 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/18ff425009144d55886c929485232e84 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/18ff425009144d55886c929485232e84 2024-11-10T19:18:04,496 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-10T19:18:04,496 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:04,496 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,496 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:04,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:04,500 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/C of bccf9fccd1002d9d7579e690ca8d056c into 18ff425009144d55886c929485232e84(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:04,500 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:04,500 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/C, priority=13, startTime=1731266283973; duration=0sec 2024-11-10T19:18:04,500 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:04,500 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:C 2024-11-10T19:18:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T19:18:04,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266344601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:04,606 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/ad87cd0ebd57420fb6b99a5cb518a012 2024-11-10T19:18:04,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266344611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/ae57bef8cb1543d4bcf5028c4cc24cba is 50, key is test_row_0/C:col10/1731266283036/Put/seqid=0 2024-11-10T19:18:04,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742278_1454 (size=12301) 2024-11-10T19:18:04,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46116 deadline: 1731266344911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:04,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46162 deadline: 1731266344921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:04,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:05,017 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/ae57bef8cb1543d4bcf5028c4cc24cba 2024-11-10T19:18:05,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/654623a793164abf9b341dffd8cf115b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/654623a793164abf9b341dffd8cf115b 2024-11-10T19:18:05,024 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/654623a793164abf9b341dffd8cf115b, entries=150, sequenceid=444, filesize=30.5 K 2024-11-10T19:18:05,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/ad87cd0ebd57420fb6b99a5cb518a012 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ad87cd0ebd57420fb6b99a5cb518a012 2024-11-10T19:18:05,031 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ad87cd0ebd57420fb6b99a5cb518a012, entries=150, sequenceid=444, filesize=12.0 K 2024-11-10T19:18:05,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/ae57bef8cb1543d4bcf5028c4cc24cba as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae57bef8cb1543d4bcf5028c4cc24cba 2024-11-10T19:18:05,035 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae57bef8cb1543d4bcf5028c4cc24cba, entries=150, sequenceid=444, filesize=12.0 K 2024-11-10T19:18:05,036 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for bccf9fccd1002d9d7579e690ca8d056c in 974ms, sequenceid=444, compaction requested=false 2024-11-10T19:18:05,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:05,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:05,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-10T19:18:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-10T19:18:05,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-10T19:18:05,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6020 sec 2024-11-10T19:18:05,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.6070 sec 2024-11-10T19:18:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:05,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T19:18:05,423 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-10T19:18:05,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:18:05,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:05,423 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:05,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:18:05,423 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:05,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:05,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:18:05,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:05,423 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:05,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:05,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111015ee34d9eb9c46f69ae5d900d850c822_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266284232/Put/seqid=0 2024-11-10T19:18:05,433 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-11-10T19:18:05,433 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:05,433 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:05,433 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:05,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:05,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742279_1455 (size=12454) 2024-11-10T19:18:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T19:18:05,591 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:05,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:05,839 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111015ee34d9eb9c46f69ae5d900d850c822_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111015ee34d9eb9c46f69ae5d900d850c822_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:05,839 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/dbf340e499ca4912baee5a5c2ae280e8, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:05,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/dbf340e499ca4912baee5a5c2ae280e8 is 175, key is test_row_0/A:col10/1731266284232/Put/seqid=0 2024-11-10T19:18:05,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742280_1456 (size=31255) 2024-11-10T19:18:05,898 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T19:18:06,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/dbf340e499ca4912baee5a5c2ae280e8 2024-11-10T19:18:06,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/07e9b189eaaf46a0a815794d27d8eaae is 50, key is test_row_0/B:col10/1731266284232/Put/seqid=0 2024-11-10T19:18:06,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742281_1457 (size=12301) 2024-11-10T19:18:06,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/07e9b189eaaf46a0a815794d27d8eaae 2024-11-10T19:18:06,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/d8fffc43934f4664ac770f428f6ae932 is 50, key is test_row_0/C:col10/1731266284232/Put/seqid=0 2024-11-10T19:18:06,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742282_1458 (size=12301) 2024-11-10T19:18:06,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/d8fffc43934f4664ac770f428f6ae932 2024-11-10T19:18:06,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/dbf340e499ca4912baee5a5c2ae280e8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/dbf340e499ca4912baee5a5c2ae280e8 2024-11-10T19:18:06,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/dbf340e499ca4912baee5a5c2ae280e8, entries=150, sequenceid=459, filesize=30.5 K 2024-11-10T19:18:06,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/07e9b189eaaf46a0a815794d27d8eaae as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/07e9b189eaaf46a0a815794d27d8eaae 2024-11-10T19:18:06,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/07e9b189eaaf46a0a815794d27d8eaae, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T19:18:06,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/d8fffc43934f4664ac770f428f6ae932 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/d8fffc43934f4664ac770f428f6ae932 2024-11-10T19:18:06,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/d8fffc43934f4664ac770f428f6ae932, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T19:18:06,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for bccf9fccd1002d9d7579e690ca8d056c in 1266ms, sequenceid=459, compaction requested=true 2024-11-10T19:18:06,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:06,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:06,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:06,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:06,689 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:06,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:06,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bccf9fccd1002d9d7579e690ca8d056c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:06,689 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:06,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:06,690 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:06,690 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:06,690 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/A is initiating minor compaction (all files) 2024-11-10T19:18:06,690 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/B is initiating minor compaction (all files) 2024-11-10T19:18:06,690 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/B in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:06,690 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/A in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:06,691 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c4c386c8b3a5499f984698003353de69, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ad87cd0ebd57420fb6b99a5cb518a012, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/07e9b189eaaf46a0a815794d27d8eaae] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=36.9 K 2024-11-10T19:18:06,691 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/0a3e64b0b545413ba504a5e5f8a52e04, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/654623a793164abf9b341dffd8cf115b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/dbf340e499ca4912baee5a5c2ae280e8] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=92.4 K 2024-11-10T19:18:06,691 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:06,691 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/0a3e64b0b545413ba504a5e5f8a52e04, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/654623a793164abf9b341dffd8cf115b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/dbf340e499ca4912baee5a5c2ae280e8] 2024-11-10T19:18:06,691 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c386c8b3a5499f984698003353de69, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731266282222 2024-11-10T19:18:06,691 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a3e64b0b545413ba504a5e5f8a52e04, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731266282222 2024-11-10T19:18:06,691 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ad87cd0ebd57420fb6b99a5cb518a012, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731266283004 2024-11-10T19:18:06,691 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654623a793164abf9b341dffd8cf115b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731266283004 2024-11-10T19:18:06,691 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e9b189eaaf46a0a815794d27d8eaae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731266284232 2024-11-10T19:18:06,691 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbf340e499ca4912baee5a5c2ae280e8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731266284232 2024-11-10T19:18:06,697 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:06,698 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100e3c5851507e49c4b41034fda32cd33e_bccf9fccd1002d9d7579e690ca8d056c store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:06,699 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:06,700 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/68c155f56f6c493eb46298f1f9a66827 is 50, key is test_row_0/B:col10/1731266284232/Put/seqid=0 2024-11-10T19:18:06,702 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100e3c5851507e49c4b41034fda32cd33e_bccf9fccd1002d9d7579e690ca8d056c, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:06,702 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100e3c5851507e49c4b41034fda32cd33e_bccf9fccd1002d9d7579e690ca8d056c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:06,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742283_1459 (size=13255) 2024-11-10T19:18:06,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742284_1460 (size=4469) 2024-11-10T19:18:06,706 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/68c155f56f6c493eb46298f1f9a66827 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/68c155f56f6c493eb46298f1f9a66827 2024-11-10T19:18:06,710 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/B of bccf9fccd1002d9d7579e690ca8d056c into 68c155f56f6c493eb46298f1f9a66827(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:06,710 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:06,710 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/B, priority=13, startTime=1731266286689; duration=0sec 2024-11-10T19:18:06,710 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:06,710 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:B 2024-11-10T19:18:06,710 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:06,711 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:06,711 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): bccf9fccd1002d9d7579e690ca8d056c/C is initiating minor compaction (all files) 2024-11-10T19:18:06,711 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bccf9fccd1002d9d7579e690ca8d056c/C in TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:06,711 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/18ff425009144d55886c929485232e84, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae57bef8cb1543d4bcf5028c4cc24cba, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/d8fffc43934f4664ac770f428f6ae932] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp, totalSize=36.9 K 2024-11-10T19:18:06,711 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ff425009144d55886c929485232e84, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731266282222 2024-11-10T19:18:06,711 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ae57bef8cb1543d4bcf5028c4cc24cba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731266283004 2024-11-10T19:18:06,712 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting d8fffc43934f4664ac770f428f6ae932, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731266284232 2024-11-10T19:18:06,717 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:06,718 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/5a86fcb4d980445a8c33aff300cb3155 is 50, key is test_row_0/C:col10/1731266284232/Put/seqid=0 2024-11-10T19:18:06,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742285_1461 (size=13255) 2024-11-10T19:18:07,106 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bccf9fccd1002d9d7579e690ca8d056c#A#compaction#378 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:07,106 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/bd58ce02bcee425aaffd06f005ddbb2c is 175, key is test_row_0/A:col10/1731266284232/Put/seqid=0 2024-11-10T19:18:07,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742286_1462 (size=32209) 2024-11-10T19:18:07,113 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/bd58ce02bcee425aaffd06f005ddbb2c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/bd58ce02bcee425aaffd06f005ddbb2c 2024-11-10T19:18:07,118 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/A of bccf9fccd1002d9d7579e690ca8d056c into bd58ce02bcee425aaffd06f005ddbb2c(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:07,118 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:07,118 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/A, priority=13, startTime=1731266286689; duration=0sec 2024-11-10T19:18:07,118 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:07,118 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:A 2024-11-10T19:18:07,128 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/5a86fcb4d980445a8c33aff300cb3155 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/5a86fcb4d980445a8c33aff300cb3155 2024-11-10T19:18:07,134 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bccf9fccd1002d9d7579e690ca8d056c/C of bccf9fccd1002d9d7579e690ca8d056c into 5a86fcb4d980445a8c33aff300cb3155(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:07,134 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bccf9fccd1002d9d7579e690ca8d056c: 2024-11-10T19:18:07,134 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., storeName=bccf9fccd1002d9d7579e690ca8d056c/C, priority=13, startTime=1731266286689; duration=0sec 2024-11-10T19:18:07,134 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:07,134 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bccf9fccd1002d9d7579e690ca8d056c:C 2024-11-10T19:18:10,947 WARN [regionserver/bdc25462c9ef:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-10T19:18:10,949 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/03e908bccb6c494db0b57eaeaf1035ae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/552886479e254fe28c15414ea1afad1d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/350ad61b808f410d9cc291b3bbf991aa, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/70263388620e4c53b69fb2e51e915892, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2ff33630c67c477fb572563bb3774b9d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5708f557702a4958b72ac4771eae55a4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/af485d6d11d24510b3583d64ad7f7ccc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f9f1ffb538fb484784a894ba0eff8f0b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/6a61d9ce3dee4c27bf9e82363c932958, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/433c0e0225bf48beab5a0a834f203750, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f7fddb24ee7245fe9666674c8063a122, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5ec8c0ada5e246d1b9331c2d6500a64b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9e89d099ee5b4f699049ebf54b1ef9c7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/301da1808ad94fad914a1a1bf24eafef, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f886159e0d5f473d911a101c40cea0b0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/49f9b696ca2e441e99898cc3137a93e6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/413a2fefa04449f7b400add315b5f30f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/a27eb07fa7694b7f9580bdc8c11272f3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9ad7828c50014ecbafbe2bcb7d8923f7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5adea20be9744d7dac6fb432cf9eeeac, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5c5bb1e2f5cd4343a4fd5535d45e761b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2f4a165fb42b4d05ae1fa3157dcc59c7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1c7acf38aab54447b19ae62b64d33726, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/442a8fd1704349e8965f785ea24a77cb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1bf9505202d6441d9ad4a093bd701642, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/0a3e64b0b545413ba504a5e5f8a52e04, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/654623a793164abf9b341dffd8cf115b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/dbf340e499ca4912baee5a5c2ae280e8] to archive 2024-11-10T19:18:10,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:18:10,951 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/03e908bccb6c494db0b57eaeaf1035ae to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/03e908bccb6c494db0b57eaeaf1035ae 2024-11-10T19:18:10,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/552886479e254fe28c15414ea1afad1d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/552886479e254fe28c15414ea1afad1d 2024-11-10T19:18:10,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/350ad61b808f410d9cc291b3bbf991aa to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/350ad61b808f410d9cc291b3bbf991aa 2024-11-10T19:18:10,954 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/70263388620e4c53b69fb2e51e915892 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/70263388620e4c53b69fb2e51e915892 2024-11-10T19:18:10,955 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2ff33630c67c477fb572563bb3774b9d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2ff33630c67c477fb572563bb3774b9d 2024-11-10T19:18:10,956 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5708f557702a4958b72ac4771eae55a4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5708f557702a4958b72ac4771eae55a4 2024-11-10T19:18:10,957 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/af485d6d11d24510b3583d64ad7f7ccc to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/af485d6d11d24510b3583d64ad7f7ccc 2024-11-10T19:18:10,957 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f9f1ffb538fb484784a894ba0eff8f0b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f9f1ffb538fb484784a894ba0eff8f0b 2024-11-10T19:18:10,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/6a61d9ce3dee4c27bf9e82363c932958 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/6a61d9ce3dee4c27bf9e82363c932958 2024-11-10T19:18:10,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/433c0e0225bf48beab5a0a834f203750 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/433c0e0225bf48beab5a0a834f203750 2024-11-10T19:18:10,961 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f7fddb24ee7245fe9666674c8063a122 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f7fddb24ee7245fe9666674c8063a122 2024-11-10T19:18:10,962 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5ec8c0ada5e246d1b9331c2d6500a64b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5ec8c0ada5e246d1b9331c2d6500a64b 2024-11-10T19:18:10,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9e89d099ee5b4f699049ebf54b1ef9c7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9e89d099ee5b4f699049ebf54b1ef9c7 2024-11-10T19:18:10,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/301da1808ad94fad914a1a1bf24eafef to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/301da1808ad94fad914a1a1bf24eafef 2024-11-10T19:18:10,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f886159e0d5f473d911a101c40cea0b0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/f886159e0d5f473d911a101c40cea0b0 2024-11-10T19:18:10,966 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/49f9b696ca2e441e99898cc3137a93e6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/49f9b696ca2e441e99898cc3137a93e6 2024-11-10T19:18:10,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/413a2fefa04449f7b400add315b5f30f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/413a2fefa04449f7b400add315b5f30f 2024-11-10T19:18:10,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/a27eb07fa7694b7f9580bdc8c11272f3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/a27eb07fa7694b7f9580bdc8c11272f3 2024-11-10T19:18:10,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9ad7828c50014ecbafbe2bcb7d8923f7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/9ad7828c50014ecbafbe2bcb7d8923f7 2024-11-10T19:18:10,970 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5adea20be9744d7dac6fb432cf9eeeac to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5adea20be9744d7dac6fb432cf9eeeac 2024-11-10T19:18:10,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5c5bb1e2f5cd4343a4fd5535d45e761b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/5c5bb1e2f5cd4343a4fd5535d45e761b 2024-11-10T19:18:10,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2f4a165fb42b4d05ae1fa3157dcc59c7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/2f4a165fb42b4d05ae1fa3157dcc59c7 2024-11-10T19:18:10,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1c7acf38aab54447b19ae62b64d33726 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1c7acf38aab54447b19ae62b64d33726 2024-11-10T19:18:10,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/442a8fd1704349e8965f785ea24a77cb to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/442a8fd1704349e8965f785ea24a77cb 2024-11-10T19:18:10,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1bf9505202d6441d9ad4a093bd701642 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/1bf9505202d6441d9ad4a093bd701642 2024-11-10T19:18:10,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/0a3e64b0b545413ba504a5e5f8a52e04 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/0a3e64b0b545413ba504a5e5f8a52e04 2024-11-10T19:18:10,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/654623a793164abf9b341dffd8cf115b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/654623a793164abf9b341dffd8cf115b 2024-11-10T19:18:10,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/dbf340e499ca4912baee5a5c2ae280e8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/dbf340e499ca4912baee5a5c2ae280e8 2024-11-10T19:18:10,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/8800ee97b0e54591ad69fdc6263fe4e8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/15def9702fe140c98de9c81c8a3c4fc6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be5951ad96de4bd29275df2cffc38cdf, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/9f6c23e66fcc49b4a46cac77ac67d9e5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ce3460c18db64a8fa5ec296e1036a7d9, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5327f33485bc43c4adc29d2a7dbb271f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/2565ab1fdfa04e84aac37fdbcb6e13b8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/231359ca583e450b8eda6f2dbb2561f3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/a7e2ee2f238648c98780dc19bf52f9e8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be7ab7f723fd41ebab1657bd472d53b7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/76588007bf6449d4ae212c3fb8939e35, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/67a78de9d6be4dffbf2c25274297a638, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/3c05eb6f1f85486882dabea44f5f591a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/cf6c33c973c2482fa3937486e419a37c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5fc9ded054b943e490c5db4d38896cd0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5cf591a3c42c41ad83961eaf9f959d1c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/54f69552d4a74c44a04363419fe90ebc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/30117294eafa4e6799089014a020f4ba, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c837cef185b440d886d96d2da4dec868, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/4d65c68072764599b3d0265485f53c87, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/da2aa2e8a1894f46aed6950d724307ea, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/82bc0ca44bec4064b560dc0555d554c8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c1fc0de4bd474421af3719dd3bf4c9e2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/68542c9b48844811a39f97d4551f318f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c4c386c8b3a5499f984698003353de69, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/bd2807376f6b44bea665c8feed259dcb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ad87cd0ebd57420fb6b99a5cb518a012, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/07e9b189eaaf46a0a815794d27d8eaae] to archive 2024-11-10T19:18:10,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:18:10,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/8800ee97b0e54591ad69fdc6263fe4e8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/8800ee97b0e54591ad69fdc6263fe4e8 2024-11-10T19:18:10,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/15def9702fe140c98de9c81c8a3c4fc6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/15def9702fe140c98de9c81c8a3c4fc6 2024-11-10T19:18:10,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be5951ad96de4bd29275df2cffc38cdf to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be5951ad96de4bd29275df2cffc38cdf 2024-11-10T19:18:10,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/9f6c23e66fcc49b4a46cac77ac67d9e5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/9f6c23e66fcc49b4a46cac77ac67d9e5 2024-11-10T19:18:10,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ce3460c18db64a8fa5ec296e1036a7d9 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ce3460c18db64a8fa5ec296e1036a7d9 2024-11-10T19:18:10,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5327f33485bc43c4adc29d2a7dbb271f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5327f33485bc43c4adc29d2a7dbb271f 2024-11-10T19:18:10,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/2565ab1fdfa04e84aac37fdbcb6e13b8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/2565ab1fdfa04e84aac37fdbcb6e13b8 2024-11-10T19:18:10,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/231359ca583e450b8eda6f2dbb2561f3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/231359ca583e450b8eda6f2dbb2561f3 2024-11-10T19:18:10,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/a7e2ee2f238648c98780dc19bf52f9e8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/a7e2ee2f238648c98780dc19bf52f9e8 2024-11-10T19:18:10,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be7ab7f723fd41ebab1657bd472d53b7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/be7ab7f723fd41ebab1657bd472d53b7 2024-11-10T19:18:10,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/76588007bf6449d4ae212c3fb8939e35 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/76588007bf6449d4ae212c3fb8939e35 2024-11-10T19:18:10,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/67a78de9d6be4dffbf2c25274297a638 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/67a78de9d6be4dffbf2c25274297a638 2024-11-10T19:18:10,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/3c05eb6f1f85486882dabea44f5f591a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/3c05eb6f1f85486882dabea44f5f591a 2024-11-10T19:18:11,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/cf6c33c973c2482fa3937486e419a37c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/cf6c33c973c2482fa3937486e419a37c 2024-11-10T19:18:11,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5fc9ded054b943e490c5db4d38896cd0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5fc9ded054b943e490c5db4d38896cd0 2024-11-10T19:18:11,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5cf591a3c42c41ad83961eaf9f959d1c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/5cf591a3c42c41ad83961eaf9f959d1c 2024-11-10T19:18:11,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/54f69552d4a74c44a04363419fe90ebc to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/54f69552d4a74c44a04363419fe90ebc 2024-11-10T19:18:11,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/30117294eafa4e6799089014a020f4ba to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/30117294eafa4e6799089014a020f4ba 2024-11-10T19:18:11,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c837cef185b440d886d96d2da4dec868 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c837cef185b440d886d96d2da4dec868 2024-11-10T19:18:11,006 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/4d65c68072764599b3d0265485f53c87 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/4d65c68072764599b3d0265485f53c87 2024-11-10T19:18:11,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/da2aa2e8a1894f46aed6950d724307ea to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/da2aa2e8a1894f46aed6950d724307ea 2024-11-10T19:18:11,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/82bc0ca44bec4064b560dc0555d554c8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/82bc0ca44bec4064b560dc0555d554c8 2024-11-10T19:18:11,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c1fc0de4bd474421af3719dd3bf4c9e2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c1fc0de4bd474421af3719dd3bf4c9e2 2024-11-10T19:18:11,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/68542c9b48844811a39f97d4551f318f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/68542c9b48844811a39f97d4551f318f 2024-11-10T19:18:11,010 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c4c386c8b3a5499f984698003353de69 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/c4c386c8b3a5499f984698003353de69 2024-11-10T19:18:11,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/bd2807376f6b44bea665c8feed259dcb to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/bd2807376f6b44bea665c8feed259dcb 2024-11-10T19:18:11,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ad87cd0ebd57420fb6b99a5cb518a012 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/ad87cd0ebd57420fb6b99a5cb518a012 2024-11-10T19:18:11,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/07e9b189eaaf46a0a815794d27d8eaae to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/07e9b189eaaf46a0a815794d27d8eaae 2024-11-10T19:18:11,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/35690ec2556c4da687cf415f8d83bb72, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/8d0394067cc1476189a07a33afa0466f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/78b275f63f064e6fb42330b332bed207, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3a76b245b3e7487facdc6b95b1ee9f68, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f1d7817656bd4adbbc76ffa922138b34, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/b15e372204da4e59bff4fde6dddad7ef, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/37b3fab531e54604aa0239243c41e76d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c0011768d7f049f388d7fcf7291aad0e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ca036378935141338af61f03e29cbbd6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae479fe6de4e45f990538e15716c7215, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/e2f60d1f936b4c5698cdd38246124808, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4f66c5f8d6664eec881d05b8d4b332a1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/0aaca40d3ece45d39d6fee1836130314, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/72f7e3919a2547ecb1a2eded34be75bf, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3faad7b080cc4659854fd22aa728f1aa, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/51610d584e754b138d216ac129d7f2c4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/eafb5d25d1a148d4be0807025bf90334, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4bbba0987d394c12ac1c89c6ef2dfe28, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/360e9b92f88d4b0d89d342680d3a7901, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/39bf37c7b7a940c7ac6a1657667dc7c1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c95d66a1005c4ff2a389702096fb1477, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4b751698b33443978a9be107d86bd56e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/849f7d4b3aa34d758bffa8e53a12d40b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4c5958ab089044599b34e2b530cdc6ed, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/18ff425009144d55886c929485232e84, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/2100322704e243e4a3cb1b2b380467f9, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae57bef8cb1543d4bcf5028c4cc24cba, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/d8fffc43934f4664ac770f428f6ae932] to archive 2024-11-10T19:18:11,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:18:11,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/35690ec2556c4da687cf415f8d83bb72 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/35690ec2556c4da687cf415f8d83bb72 2024-11-10T19:18:11,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/8d0394067cc1476189a07a33afa0466f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/8d0394067cc1476189a07a33afa0466f 2024-11-10T19:18:11,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/78b275f63f064e6fb42330b332bed207 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/78b275f63f064e6fb42330b332bed207 2024-11-10T19:18:11,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3a76b245b3e7487facdc6b95b1ee9f68 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3a76b245b3e7487facdc6b95b1ee9f68 2024-11-10T19:18:11,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f1d7817656bd4adbbc76ffa922138b34 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f1d7817656bd4adbbc76ffa922138b34 2024-11-10T19:18:11,022 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/b15e372204da4e59bff4fde6dddad7ef to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/b15e372204da4e59bff4fde6dddad7ef 2024-11-10T19:18:11,024 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/37b3fab531e54604aa0239243c41e76d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/37b3fab531e54604aa0239243c41e76d 2024-11-10T19:18:11,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c0011768d7f049f388d7fcf7291aad0e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c0011768d7f049f388d7fcf7291aad0e 2024-11-10T19:18:11,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ca036378935141338af61f03e29cbbd6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ca036378935141338af61f03e29cbbd6 2024-11-10T19:18:11,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae479fe6de4e45f990538e15716c7215 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae479fe6de4e45f990538e15716c7215 2024-11-10T19:18:11,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/e2f60d1f936b4c5698cdd38246124808 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/e2f60d1f936b4c5698cdd38246124808 2024-11-10T19:18:11,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4f66c5f8d6664eec881d05b8d4b332a1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4f66c5f8d6664eec881d05b8d4b332a1 2024-11-10T19:18:11,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/0aaca40d3ece45d39d6fee1836130314 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/0aaca40d3ece45d39d6fee1836130314 2024-11-10T19:18:11,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/72f7e3919a2547ecb1a2eded34be75bf to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/72f7e3919a2547ecb1a2eded34be75bf 2024-11-10T19:18:11,032 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3faad7b080cc4659854fd22aa728f1aa to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/3faad7b080cc4659854fd22aa728f1aa 2024-11-10T19:18:11,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/51610d584e754b138d216ac129d7f2c4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/51610d584e754b138d216ac129d7f2c4 2024-11-10T19:18:11,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/eafb5d25d1a148d4be0807025bf90334 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/eafb5d25d1a148d4be0807025bf90334 2024-11-10T19:18:11,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4bbba0987d394c12ac1c89c6ef2dfe28 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4bbba0987d394c12ac1c89c6ef2dfe28 2024-11-10T19:18:11,036 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/360e9b92f88d4b0d89d342680d3a7901 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/360e9b92f88d4b0d89d342680d3a7901 2024-11-10T19:18:11,037 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/39bf37c7b7a940c7ac6a1657667dc7c1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/39bf37c7b7a940c7ac6a1657667dc7c1 2024-11-10T19:18:11,038 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c95d66a1005c4ff2a389702096fb1477 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/c95d66a1005c4ff2a389702096fb1477 2024-11-10T19:18:11,039 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4b751698b33443978a9be107d86bd56e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4b751698b33443978a9be107d86bd56e 2024-11-10T19:18:11,040 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/849f7d4b3aa34d758bffa8e53a12d40b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/849f7d4b3aa34d758bffa8e53a12d40b 2024-11-10T19:18:11,041 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4c5958ab089044599b34e2b530cdc6ed to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/4c5958ab089044599b34e2b530cdc6ed 2024-11-10T19:18:11,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/18ff425009144d55886c929485232e84 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/18ff425009144d55886c929485232e84 2024-11-10T19:18:11,043 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/2100322704e243e4a3cb1b2b380467f9 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/2100322704e243e4a3cb1b2b380467f9 2024-11-10T19:18:11,044 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae57bef8cb1543d4bcf5028c4cc24cba to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/ae57bef8cb1543d4bcf5028c4cc24cba 2024-11-10T19:18:11,044 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bdc25462c9ef:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/d8fffc43934f4664ac770f428f6ae932 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/d8fffc43934f4664ac770f428f6ae932 2024-11-10T19:18:13,472 INFO [Thread-1580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1580. 2024-11-10T19:18:13,472 DEBUG [Thread-1580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:13,472 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:13,472 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:13,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:13,492 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-11-10T19:18:13,493 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:13,493 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:13,493 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:13,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:13,503 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-11-10T19:18:13,503 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:13,503 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:13,503 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:13,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:13,503 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T19:18:13,503 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-10T19:18:13,503 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-10T19:18:13,503 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 750 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2250 rows 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 761 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2276 rows 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 762 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2286 rows 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 763 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2288 rows 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 763 2024-11-10T19:18:13,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2282 rows 2024-11-10T19:18:13,504 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:18:13,504 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:18:13,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:13,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:13,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:13,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T19:18:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T19:18:13,508 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266293507"}]},"ts":"1731266293507"} 2024-11-10T19:18:13,509 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T19:18:13,509 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T19:18:13,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T19:18:13,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, UNASSIGN}] 2024-11-10T19:18:13,512 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, UNASSIGN 2024-11-10T19:18:13,513 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=bccf9fccd1002d9d7579e690ca8d056c, regionState=CLOSING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:13,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, UNASSIGN because future has completed 2024-11-10T19:18:13,514 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T19:18:13,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:18:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T19:18:13,667 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:13,667 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T19:18:13,667 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing bccf9fccd1002d9d7579e690ca8d056c, disabling compactions & flushes 2024-11-10T19:18:13,667 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:13,667 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:13,667 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. after waiting 0 ms 2024-11-10T19:18:13,667 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:13,668 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing bccf9fccd1002d9d7579e690ca8d056c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T19:18:13,668 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=A 2024-11-10T19:18:13,668 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:13,668 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=B 2024-11-10T19:18:13,669 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:13,669 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bccf9fccd1002d9d7579e690ca8d056c, store=C 2024-11-10T19:18:13,669 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:13,678 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f58c4764b683416ea01a7b6a8f3f4704_bccf9fccd1002d9d7579e690ca8d056c is 50, key is test_row_0/A:col10/1731266293471/Put/seqid=0 2024-11-10T19:18:13,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742287_1463 (size=9914) 2024-11-10T19:18:13,684 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:13,689 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f58c4764b683416ea01a7b6a8f3f4704_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f58c4764b683416ea01a7b6a8f3f4704_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:13,690 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/8c41282995174de49cb842926e0d02ce, store: [table=TestAcidGuarantees family=A region=bccf9fccd1002d9d7579e690ca8d056c] 2024-11-10T19:18:13,691 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/8c41282995174de49cb842926e0d02ce is 175, key is test_row_0/A:col10/1731266293471/Put/seqid=0 2024-11-10T19:18:13,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742288_1464 (size=22561) 2024-11-10T19:18:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T19:18:14,095 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=469, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/8c41282995174de49cb842926e0d02ce 2024-11-10T19:18:14,100 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/553aae06bb5743339a5fdd15e1a5c3ba is 50, key is test_row_0/B:col10/1731266293471/Put/seqid=0 2024-11-10T19:18:14,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742289_1465 (size=9857) 2024-11-10T19:18:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T19:18:14,503 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/553aae06bb5743339a5fdd15e1a5c3ba 2024-11-10T19:18:14,510 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/f75467f151df42638f9ab27b84978c49 is 50, key is test_row_0/C:col10/1731266293471/Put/seqid=0 2024-11-10T19:18:14,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742290_1466 (size=9857) 2024-11-10T19:18:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T19:18:14,926 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/f75467f151df42638f9ab27b84978c49 2024-11-10T19:18:14,930 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/A/8c41282995174de49cb842926e0d02ce as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/8c41282995174de49cb842926e0d02ce 2024-11-10T19:18:14,934 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/8c41282995174de49cb842926e0d02ce, entries=100, sequenceid=469, filesize=22.0 K 2024-11-10T19:18:14,935 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/B/553aae06bb5743339a5fdd15e1a5c3ba as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/553aae06bb5743339a5fdd15e1a5c3ba 2024-11-10T19:18:14,939 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/553aae06bb5743339a5fdd15e1a5c3ba, entries=100, sequenceid=469, filesize=9.6 K 2024-11-10T19:18:14,940 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/.tmp/C/f75467f151df42638f9ab27b84978c49 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f75467f151df42638f9ab27b84978c49 2024-11-10T19:18:14,944 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f75467f151df42638f9ab27b84978c49, entries=100, sequenceid=469, filesize=9.6 K 2024-11-10T19:18:14,945 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for bccf9fccd1002d9d7579e690ca8d056c in 1278ms, sequenceid=469, compaction requested=false 2024-11-10T19:18:14,951 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/recovered.edits/472.seqid, newMaxSeqId=472, maxSeqId=4 2024-11-10T19:18:14,952 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. 2024-11-10T19:18:14,952 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for bccf9fccd1002d9d7579e690ca8d056c: Waiting for close lock at 1731266293667Running coprocessor pre-close hooks at 1731266293667Disabling compacts and flushes for region at 1731266293667Disabling writes for close at 1731266293667Obtaining lock to block concurrent updates at 1731266293668 (+1 ms)Preparing flush snapshotting stores in bccf9fccd1002d9d7579e690ca8d056c at 1731266293668Finished memstore snapshotting TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731266293669 (+1 ms)Flushing stores of TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c. at 1731266293670 (+1 ms)Flushing bccf9fccd1002d9d7579e690ca8d056c/A: creating writer at 1731266293670Flushing mob file bccf9fccd1002d9d7579e690ca8d056c/A: appending metadata at 1731266293678 (+8 ms)Flushing mob file bccf9fccd1002d9d7579e690ca8d056c/A: closing flushed file at 1731266293678Flushing bccf9fccd1002d9d7579e690ca8d056c/A: appending metadata at 1731266293690 (+12 ms)Flushing bccf9fccd1002d9d7579e690ca8d056c/A: closing flushed file at 1731266293690Flushing bccf9fccd1002d9d7579e690ca8d056c/B: creating writer at 1731266294098 (+408 ms)Flushing bccf9fccd1002d9d7579e690ca8d056c/B: appending metadata at 1731266294100 (+2 ms)Flushing bccf9fccd1002d9d7579e690ca8d056c/B: closing flushed file at 1731266294100Flushing bccf9fccd1002d9d7579e690ca8d056c/C: creating writer at 1731266294507 (+407 ms)Flushing bccf9fccd1002d9d7579e690ca8d056c/C: appending metadata at 1731266294510 (+3 ms)Flushing bccf9fccd1002d9d7579e690ca8d056c/C: closing flushed file at 1731266294510Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f9efa73: reopening flushed file at 1731266294929 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d8eefdb: reopening flushed file at 1731266294934 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b9241c6: reopening flushed file at 1731266294939 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for bccf9fccd1002d9d7579e690ca8d056c in 1278ms, sequenceid=469, compaction requested=false at 1731266294945 (+6 ms)Writing region close event to WAL at 1731266294948 (+3 ms)Running coprocessor post-close hooks at 1731266294952 (+4 ms)Closed at 1731266294952 2024-11-10T19:18:14,954 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:14,954 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=bccf9fccd1002d9d7579e690ca8d056c, regionState=CLOSED 2024-11-10T19:18:14,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:18:14,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-10T19:18:14,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure bccf9fccd1002d9d7579e690ca8d056c, server=bdc25462c9ef,34487,1731266167729 in 1.4420 sec 2024-11-10T19:18:14,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-10T19:18:14,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bccf9fccd1002d9d7579e690ca8d056c, UNASSIGN in 1.4470 sec 2024-11-10T19:18:14,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-10T19:18:14,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4490 sec 2024-11-10T19:18:14,962 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266294962"}]},"ts":"1731266294962"} 2024-11-10T19:18:14,963 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T19:18:14,963 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T19:18:14,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4590 sec 2024-11-10T19:18:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T19:18:15,653 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:15,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T19:18:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:15,656 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T19:18:15,657 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:15,658 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,660 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/recovered.edits] 2024-11-10T19:18:15,661 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/8c41282995174de49cb842926e0d02ce to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/8c41282995174de49cb842926e0d02ce 2024-11-10T19:18:15,662 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/bd58ce02bcee425aaffd06f005ddbb2c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/A/bd58ce02bcee425aaffd06f005ddbb2c 2024-11-10T19:18:15,664 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/553aae06bb5743339a5fdd15e1a5c3ba to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/553aae06bb5743339a5fdd15e1a5c3ba 2024-11-10T19:18:15,664 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/68c155f56f6c493eb46298f1f9a66827 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/B/68c155f56f6c493eb46298f1f9a66827 2024-11-10T19:18:15,666 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/5a86fcb4d980445a8c33aff300cb3155 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/5a86fcb4d980445a8c33aff300cb3155 2024-11-10T19:18:15,666 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f75467f151df42638f9ab27b84978c49 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/C/f75467f151df42638f9ab27b84978c49 2024-11-10T19:18:15,668 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/recovered.edits/472.seqid to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c/recovered.edits/472.seqid 2024-11-10T19:18:15,668 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,668 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T19:18:15,668 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T19:18:15,669 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T19:18:15,671 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111015ee34d9eb9c46f69ae5d900d850c822_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111015ee34d9eb9c46f69ae5d900d850c822_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,672 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101cda226582484e57a65a0be532237ff1_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101cda226582484e57a65a0be532237ff1_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,673 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021df7ce12d5649729c2bcd4a133da3c4_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021df7ce12d5649729c2bcd4a133da3c4_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,674 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104464676aa5974b75b8eeb2f78256903b_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104464676aa5974b75b8eeb2f78256903b_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,675 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111044fb7d7cb6ae4b2fac2e758f0ea35072_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111044fb7d7cb6ae4b2fac2e758f0ea35072_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,676 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110465140f7c3a94211acf851aba20de369_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110465140f7c3a94211acf851aba20de369_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,677 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046dd9dfd100f4f08902f987a76c923a7_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046dd9dfd100f4f08902f987a76c923a7_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,678 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105c76f044539c4535ae37f90fd03a8088_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105c76f044539c4535ae37f90fd03a8088_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,678 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105fc3cd84d4044f1ebad4863cd72da45d_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105fc3cd84d4044f1ebad4863cd72da45d_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,679 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110695faef98d7b421ca495fa1a8a86f7c2_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110695faef98d7b421ca495fa1a8a86f7c2_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,680 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111077a3c90ca0fd41599a2015558dae3042_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111077a3c90ca0fd41599a2015558dae3042_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,681 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107c62046651f74985bc5c5d88e709c361_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107c62046651f74985bc5c5d88e709c361_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,682 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108635f3c27f564190ae43c713d7692e02_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108635f3c27f564190ae43c713d7692e02_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,683 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b09d583be3f64f3dbbc2ddeba3fa3ec1_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b09d583be3f64f3dbbc2ddeba3fa3ec1_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,684 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b2143be700dc416da9f0564c72322e17_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b2143be700dc416da9f0564c72322e17_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,685 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be59d6ac1dc24ce5bd3d62706f480e98_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be59d6ac1dc24ce5bd3d62706f480e98_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,686 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bfd0d2cb29544f71b5f86a86da75b514_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bfd0d2cb29544f71b5f86a86da75b514_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,687 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e1b024cff8034ef6a84e8c8402fc5de1_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e1b024cff8034ef6a84e8c8402fc5de1_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,688 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e7ab8bb5c73e43bf9888a9b68bd1a763_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e7ab8bb5c73e43bf9888a9b68bd1a763_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,689 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f58c4764b683416ea01a7b6a8f3f4704_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f58c4764b683416ea01a7b6a8f3f4704_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,690 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f5e1f3a8b2b24b3d9f48ebac7a1c4874_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f5e1f3a8b2b24b3d9f48ebac7a1c4874_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,691 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ffe29aa02600413bafc8cad0ccd1d70b_bccf9fccd1002d9d7579e690ca8d056c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ffe29aa02600413bafc8cad0ccd1d70b_bccf9fccd1002d9d7579e690ca8d056c 2024-11-10T19:18:15,692 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T19:18:15,694 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:15,696 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T19:18:15,698 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T19:18:15,700 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:15,700 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T19:18:15,700 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731266295700"}]},"ts":"9223372036854775807"} 2024-11-10T19:18:15,702 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T19:18:15,702 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bccf9fccd1002d9d7579e690ca8d056c, NAME => 'TestAcidGuarantees,,1731266261274.bccf9fccd1002d9d7579e690ca8d056c.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T19:18:15,702 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T19:18:15,702 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731266295702"}]},"ts":"9223372036854775807"} 2024-11-10T19:18:15,704 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T19:18:15,705 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:15,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-10T19:18:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T19:18:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T19:18:15,762 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:15,774 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=805 (was 780) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2177 (was 1182) - AvailableMemoryMB LEAK? - 2024-11-10T19:18:15,784 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=805, ProcessCount=11, AvailableMemoryMB=2177 2024-11-10T19:18:15,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T19:18:15,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:18:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:15,788 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T19:18:15,788 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:15,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-10T19:18:15,789 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T19:18:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T19:18:15,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742291_1467 (size=1061) 2024-11-10T19:18:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T19:18:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T19:18:16,198 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0 2024-11-10T19:18:16,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742292_1468 (size=53) 2024-11-10T19:18:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T19:18:16,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:18:16,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fef56f8ea94caf0284a0ef9638b76a82, disabling compactions & flushes 2024-11-10T19:18:16,608 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:16,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:16,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. after waiting 0 ms 2024-11-10T19:18:16,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:16,608 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:16,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fef56f8ea94caf0284a0ef9638b76a82: Waiting for close lock at 1731266296608Disabling compacts and flushes for region at 1731266296608Disabling writes for close at 1731266296608Writing region close event to WAL at 1731266296608Closed at 1731266296608 2024-11-10T19:18:16,609 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T19:18:16,609 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731266296609"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731266296609"}]},"ts":"1731266296609"} 2024-11-10T19:18:16,610 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T19:18:16,611 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T19:18:16,611 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266296611"}]},"ts":"1731266296611"} 2024-11-10T19:18:16,612 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T19:18:16,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fef56f8ea94caf0284a0ef9638b76a82, ASSIGN}] 2024-11-10T19:18:16,613 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fef56f8ea94caf0284a0ef9638b76a82, ASSIGN 2024-11-10T19:18:16,614 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fef56f8ea94caf0284a0ef9638b76a82, ASSIGN; state=OFFLINE, location=bdc25462c9ef,34487,1731266167729; forceNewPlan=false, retain=false 2024-11-10T19:18:16,764 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=fef56f8ea94caf0284a0ef9638b76a82, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:16,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fef56f8ea94caf0284a0ef9638b76a82, ASSIGN because future has completed 2024-11-10T19:18:16,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:18:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T19:18:16,924 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:16,925 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:18:16,925 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,926 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:18:16,926 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,926 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,928 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,930 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:18:16,931 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fef56f8ea94caf0284a0ef9638b76a82 columnFamilyName A 2024-11-10T19:18:16,931 DEBUG [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:16,931 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] regionserver.HStore(327): Store=fef56f8ea94caf0284a0ef9638b76a82/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:18:16,932 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,933 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:18:16,933 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fef56f8ea94caf0284a0ef9638b76a82 columnFamilyName B 2024-11-10T19:18:16,933 DEBUG [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:16,934 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] regionserver.HStore(327): Store=fef56f8ea94caf0284a0ef9638b76a82/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:18:16,934 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,936 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:18:16,936 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fef56f8ea94caf0284a0ef9638b76a82 columnFamilyName C 2024-11-10T19:18:16,936 DEBUG [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:16,936 INFO [StoreOpener-fef56f8ea94caf0284a0ef9638b76a82-1 {}] regionserver.HStore(327): Store=fef56f8ea94caf0284a0ef9638b76a82/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:18:16,937 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:16,937 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,937 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,938 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,939 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,939 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,940 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:18:16,942 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,944 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:18:16,945 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened fef56f8ea94caf0284a0ef9638b76a82; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68797836, jitterRate=0.025167644023895264}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:18:16,945 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:16,946 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for fef56f8ea94caf0284a0ef9638b76a82: Running coprocessor pre-open hook at 1731266296926Writing region info on filesystem at 1731266296926Initializing all the Stores at 1731266296927 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266296927Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266296928 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266296928Cleaning up temporary data from old regions at 1731266296939 (+11 ms)Running coprocessor post-open hooks at 1731266296945 (+6 ms)Region opened successfully at 1731266296946 (+1 ms) 2024-11-10T19:18:16,948 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., pid=108, masterSystemTime=1731266296919 2024-11-10T19:18:16,950 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:16,950 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:16,951 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=fef56f8ea94caf0284a0ef9638b76a82, regionState=OPEN, openSeqNum=2, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:16,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:18:16,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T19:18:16,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 in 186 msec 2024-11-10T19:18:16,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-10T19:18:16,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fef56f8ea94caf0284a0ef9638b76a82, ASSIGN in 343 msec 2024-11-10T19:18:16,958 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T19:18:16,958 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266296958"}]},"ts":"1731266296958"} 2024-11-10T19:18:16,959 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T19:18:16,960 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T19:18:16,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-10T19:18:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-10T19:18:17,931 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:17,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,933 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17c75010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,934 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1670b130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,935 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:17,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:17,937 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T19:18:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is bdc25462c9ef,46761,1731266166843 2024-11-10T19:18:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d52741 2024-11-10T19:18:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T19:18:17,939 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T19:18:17,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,942 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd28a00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,943 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b4aa393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,944 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c4b509, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,945 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db00d2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,947 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b138865, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,948 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143835c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,950 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14242be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576883f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e43aa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626b8a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,957 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f17e2f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,959 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8bf85c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,961 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f0e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,963 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9aafc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,966 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62811f33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,967 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@171b0de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,971 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603eb48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,973 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4342c883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:17,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:17,981 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5577f34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:17,983 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@722b7809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:17,985 DEBUG [Thread-2094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,985 DEBUG [Thread-2098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,985 DEBUG [Thread-2105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,986 DEBUG [Thread-2107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:17,986 DEBUG [Thread-2109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:17,987 DEBUG [Thread-2092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:17,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:17,988 DEBUG [Thread-2107 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:17,989 DEBUG [Thread-2107 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:17,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:17,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:17,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:17,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:17,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:17,992 DEBUG [Thread-2090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:17,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:17,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:17,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:17,998 DEBUG [Thread-2101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,998 DEBUG [Thread-2103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:17,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-10T19:18:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T19:18:18,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:18,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:18,002 DEBUG [Thread-2096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:18,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:18,010 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2] 2024-11-10T19:18:18,013 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:18,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:18,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:18:18,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:18,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:18,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:18,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:18,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:18,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:18,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/c6472fb7039d49558c4061ed42e64975 is 50, key is test_row_0/A:col10/1731266297996/Put/seqid=0 2024-11-10T19:18:18,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742293_1469 (size=12001) 2024-11-10T19:18:18,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266358043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731266298035, max timestamp=1731266298058Num uniques 859; 2024-11-10T19:18:18,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T19:18:18,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731266298035, max timestamp=1731266298058Num uniques 859; 2024-11-10T19:18:18,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T19:18:18,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731266298035, max timestamp=1731266298058Num uniques 859; 2024-11-10T19:18:18,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T19:18:18,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266358092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266358094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266358097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266358098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T19:18:18,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T19:18:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266358171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266358212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266358212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266358212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266358223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T19:18:18,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:18,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T19:18:18,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266358391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266358431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266358431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266358431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/c6472fb7039d49558c4061ed42e64975 2024-11-10T19:18:18,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266358441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T19:18:18,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:18,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/c86d5cb044fa4417893552682ea8deb5 is 50, key is test_row_0/B:col10/1731266297996/Put/seqid=0 2024-11-10T19:18:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742294_1470 (size=12001) 2024-11-10T19:18:18,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/c86d5cb044fa4417893552682ea8deb5 2024-11-10T19:18:18,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f293edd8fe1144a8acdfa30ef4690071 is 50, key is test_row_0/C:col10/1731266297996/Put/seqid=0 2024-11-10T19:18:18,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742295_1471 (size=12001) 2024-11-10T19:18:18,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T19:18:18,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:18,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T19:18:18,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266358711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266358741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266358752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266358752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266358761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:18,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T19:18:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:18,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f293edd8fe1144a8acdfa30ef4690071 2024-11-10T19:18:18,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/c6472fb7039d49558c4061ed42e64975 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c6472fb7039d49558c4061ed42e64975 2024-11-10T19:18:18,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c6472fb7039d49558c4061ed42e64975, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T19:18:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/c86d5cb044fa4417893552682ea8deb5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/c86d5cb044fa4417893552682ea8deb5 2024-11-10T19:18:18,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/c86d5cb044fa4417893552682ea8deb5, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T19:18:18,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f293edd8fe1144a8acdfa30ef4690071 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f293edd8fe1144a8acdfa30ef4690071 2024-11-10T19:18:18,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f293edd8fe1144a8acdfa30ef4690071, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T19:18:18,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for fef56f8ea94caf0284a0ef9638b76a82 in 916ms, sequenceid=12, compaction requested=false 2024-11-10T19:18:18,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:18,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T19:18:18,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:18,937 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-10T19:18:18,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/72594e4bbafe48ca8714b22ac33d3ddc is 50, key is test_row_0/A:col10/1731266298096/Put/seqid=0 2024-11-10T19:18:18,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742296_1472 (size=12001) 2024-11-10T19:18:18,946 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/72594e4bbafe48ca8714b22ac33d3ddc 2024-11-10T19:18:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6e9dab23089c44dda3969074f6f958d6 is 50, key is test_row_0/B:col10/1731266298096/Put/seqid=0 2024-11-10T19:18:18,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742297_1473 (size=12001) 2024-11-10T19:18:18,958 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6e9dab23089c44dda3969074f6f958d6 2024-11-10T19:18:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/39b5e5a227e3471087a30a46bb7ea8cc is 50, key is test_row_0/C:col10/1731266298096/Put/seqid=0 2024-11-10T19:18:18,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742298_1474 (size=12001) 2024-11-10T19:18:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T19:18:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:19,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:19,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266359271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266359271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266359272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266359273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266359281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,370 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/39b5e5a227e3471087a30a46bb7ea8cc 2024-11-10T19:18:19,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/72594e4bbafe48ca8714b22ac33d3ddc as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/72594e4bbafe48ca8714b22ac33d3ddc 2024-11-10T19:18:19,380 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/72594e4bbafe48ca8714b22ac33d3ddc, entries=150, sequenceid=46, filesize=11.7 K 2024-11-10T19:18:19,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6e9dab23089c44dda3969074f6f958d6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e9dab23089c44dda3969074f6f958d6 2024-11-10T19:18:19,383 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e9dab23089c44dda3969074f6f958d6, entries=150, sequenceid=46, filesize=11.7 K 2024-11-10T19:18:19,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/39b5e5a227e3471087a30a46bb7ea8cc as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/39b5e5a227e3471087a30a46bb7ea8cc 2024-11-10T19:18:19,387 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/39b5e5a227e3471087a30a46bb7ea8cc, entries=150, sequenceid=46, filesize=11.7 K 2024-11-10T19:18:19,387 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for fef56f8ea94caf0284a0ef9638b76a82 in 450ms, sequenceid=46, compaction requested=false 2024-11-10T19:18:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-10T19:18:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-10T19:18:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:19,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T19:18:19,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:19,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:19,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:19,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:19,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:19,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:19,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T19:18:19,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3760 sec 2024-11-10T19:18:19,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.3960 sec 2024-11-10T19:18:19,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/5c3cff7352c349cf979f26a7ab6df4a6 is 50, key is test_row_0/A:col10/1731266299272/Put/seqid=0 2024-11-10T19:18:19,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742299_1475 (size=14341) 2024-11-10T19:18:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731266299421, max timestamp=1731266299463Num uniques 859; 2024-11-10T19:18:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731266299421, max timestamp=1731266299463Num uniques 859; 2024-11-10T19:18:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T19:18:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8470, suffix heap size=117064,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=185 2024-11-10T19:18:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731266299421, max timestamp=1731266299463Num uniques 859; 2024-11-10T19:18:19,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T19:18:19,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266359573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266359574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266359588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266359691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266359691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266359701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/5c3cff7352c349cf979f26a7ab6df4a6 2024-11-10T19:18:19,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/d8c16258c89c4294b4e606db5fb2b88d is 50, key is test_row_0/B:col10/1731266299272/Put/seqid=0 2024-11-10T19:18:19,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742300_1476 (size=12001) 2024-11-10T19:18:19,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266359911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266359911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266359912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T19:18:20,161 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:20,161 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-10T19:18:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T19:18:20,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:20,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:20,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:20,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/d8c16258c89c4294b4e606db5fb2b88d 2024-11-10T19:18:20,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/bf17b9edb47c49638d2ce043b83acc34 is 50, key is test_row_0/C:col10/1731266299272/Put/seqid=0 2024-11-10T19:18:20,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742301_1477 (size=12001) 2024-11-10T19:18:20,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266360221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266360231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266360231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T19:18:20,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266360291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,298 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in fef56f8ea94caf0284a0ef9638b76a82 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266360301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,313 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in fef56f8ea94caf0284a0ef9638b76a82 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,477 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T19:18:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T19:18:20,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/bf17b9edb47c49638d2ce043b83acc34 2024-11-10T19:18:20,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/5c3cff7352c349cf979f26a7ab6df4a6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/5c3cff7352c349cf979f26a7ab6df4a6 2024-11-10T19:18:20,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/5c3cff7352c349cf979f26a7ab6df4a6, entries=200, sequenceid=58, filesize=14.0 K 2024-11-10T19:18:20,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/d8c16258c89c4294b4e606db5fb2b88d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d8c16258c89c4294b4e606db5fb2b88d 2024-11-10T19:18:20,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d8c16258c89c4294b4e606db5fb2b88d, entries=150, sequenceid=58, filesize=11.7 K 2024-11-10T19:18:20,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/bf17b9edb47c49638d2ce043b83acc34 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/bf17b9edb47c49638d2ce043b83acc34 2024-11-10T19:18:20,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/bf17b9edb47c49638d2ce043b83acc34, entries=150, sequenceid=58, filesize=11.7 K 2024-11-10T19:18:20,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=124.17 KB/127146 for fef56f8ea94caf0284a0ef9638b76a82 in 1251ms, sequenceid=58, compaction requested=true 2024-11-10T19:18:20,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:20,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:20,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:20,641 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:20,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:20,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:20,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:20,641 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:20,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:20,642 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:20,642 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:20,642 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/B is initiating minor compaction (all files) 2024-11-10T19:18:20,642 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/A is initiating minor compaction (all files) 2024-11-10T19:18:20,642 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/B in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,642 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/A in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,642 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/c86d5cb044fa4417893552682ea8deb5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e9dab23089c44dda3969074f6f958d6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d8c16258c89c4294b4e606db5fb2b88d] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=35.2 K 2024-11-10T19:18:20,642 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c6472fb7039d49558c4061ed42e64975, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/72594e4bbafe48ca8714b22ac33d3ddc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/5c3cff7352c349cf979f26a7ab6df4a6] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=37.4 K 2024-11-10T19:18:20,642 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c86d5cb044fa4417893552682ea8deb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731266297996 2024-11-10T19:18:20,642 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c6472fb7039d49558c4061ed42e64975, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731266297996 2024-11-10T19:18:20,642 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 72594e4bbafe48ca8714b22ac33d3ddc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731266298091 2024-11-10T19:18:20,642 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9dab23089c44dda3969074f6f958d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731266298091 2024-11-10T19:18:20,643 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c3cff7352c349cf979f26a7ab6df4a6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731266299269 2024-11-10T19:18:20,643 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8c16258c89c4294b4e606db5fb2b88d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731266299269 2024-11-10T19:18:20,649 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:20,649 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/7640dcd8b25344b0a91f246b52b6542d is 50, key is test_row_0/B:col10/1731266299272/Put/seqid=0 2024-11-10T19:18:20,661 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#A#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:20,662 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/cb5ee0e9dbe54590abaf028a73b2bb7d is 50, key is test_row_0/A:col10/1731266299272/Put/seqid=0 2024-11-10T19:18:20,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742302_1478 (size=12104) 2024-11-10T19:18:20,676 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/7640dcd8b25344b0a91f246b52b6542d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7640dcd8b25344b0a91f246b52b6542d 2024-11-10T19:18:20,680 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/B of fef56f8ea94caf0284a0ef9638b76a82 into 7640dcd8b25344b0a91f246b52b6542d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:20,680 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:20,680 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/B, priority=13, startTime=1731266300641; duration=0sec 2024-11-10T19:18:20,680 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:20,680 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:B 2024-11-10T19:18:20,680 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:20,681 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:20,681 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/C is initiating minor compaction (all files) 2024-11-10T19:18:20,681 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/C in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,681 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f293edd8fe1144a8acdfa30ef4690071, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/39b5e5a227e3471087a30a46bb7ea8cc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/bf17b9edb47c49638d2ce043b83acc34] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=35.2 K 2024-11-10T19:18:20,682 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f293edd8fe1144a8acdfa30ef4690071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731266297996 2024-11-10T19:18:20,682 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b5e5a227e3471087a30a46bb7ea8cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731266298091 2024-11-10T19:18:20,682 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf17b9edb47c49638d2ce043b83acc34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731266299269 2024-11-10T19:18:20,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742303_1479 (size=12104) 2024-11-10T19:18:20,699 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:20,699 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/2416ef3ca4e6421a93906998cd7db5e5 is 50, key is test_row_0/C:col10/1731266299272/Put/seqid=0 2024-11-10T19:18:20,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742304_1480 (size=12104) 2024-11-10T19:18:20,707 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/2416ef3ca4e6421a93906998cd7db5e5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2416ef3ca4e6421a93906998cd7db5e5 2024-11-10T19:18:20,711 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/C of fef56f8ea94caf0284a0ef9638b76a82 into 2416ef3ca4e6421a93906998cd7db5e5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:20,711 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:20,711 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/C, priority=13, startTime=1731266300641; duration=0sec 2024-11-10T19:18:20,711 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:20,711 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:C 2024-11-10T19:18:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:20,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=130.88 KB heapSize=368.09 KB 2024-11-10T19:18:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:20,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/cb8312285f2c400b8f4aa9251bf4ac8d is 50, key is test_row_0/A:col10/1731266299557/Put/seqid=0 2024-11-10T19:18:20,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742305_1481 (size=14341) 2024-11-10T19:18:20,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266360773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266360776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266360780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T19:18:20,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266360892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266360892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266360892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:20,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:20,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,098 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/cb5ee0e9dbe54590abaf028a73b2bb7d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb5ee0e9dbe54590abaf028a73b2bb7d 2024-11-10T19:18:21,102 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/A of fef56f8ea94caf0284a0ef9638b76a82 into cb5ee0e9dbe54590abaf028a73b2bb7d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:21,102 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:21,102 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/A, priority=13, startTime=1731266300641; duration=0sec 2024-11-10T19:18:21,102 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:21,102 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:A 2024-11-10T19:18:21,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266361111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:21,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266361111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266361111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:21,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/cb8312285f2c400b8f4aa9251bf4ac8d 2024-11-10T19:18:21,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6e85f96b27584589a8ac19ad0ec008f7 is 50, key is test_row_0/B:col10/1731266299557/Put/seqid=0 2024-11-10T19:18:21,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742306_1482 (size=12001) 2024-11-10T19:18:21,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:21,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:21,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T19:18:21,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:21,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:21,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266361431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266361431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266361431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:21,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6e85f96b27584589a8ac19ad0ec008f7 2024-11-10T19:18:21,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/18da0a423f6749b4a91010cbd95acd7a is 50, key is test_row_0/C:col10/1731266299557/Put/seqid=0 2024-11-10T19:18:21,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742307_1483 (size=12001) 2024-11-10T19:18:21,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:21,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:21,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:21,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:21,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:21,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266361951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:21,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266361951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:21,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266361951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:21,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.42 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/18da0a423f6749b4a91010cbd95acd7a 2024-11-10T19:18:21,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/cb8312285f2c400b8f4aa9251bf4ac8d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb8312285f2c400b8f4aa9251bf4ac8d 2024-11-10T19:18:21,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb8312285f2c400b8f4aa9251bf4ac8d, entries=200, sequenceid=95, filesize=14.0 K 2024-11-10T19:18:21,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6e85f96b27584589a8ac19ad0ec008f7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e85f96b27584589a8ac19ad0ec008f7 2024-11-10T19:18:22,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e85f96b27584589a8ac19ad0ec008f7, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T19:18:22,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/18da0a423f6749b4a91010cbd95acd7a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/18da0a423f6749b4a91010cbd95acd7a 2024-11-10T19:18:22,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:22,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:22,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/18da0a423f6749b4a91010cbd95acd7a, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T19:18:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:22,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~130.88 KB/134016, heapSize ~368.05 KB/376880, currentSize=53.67 KB/54960 for fef56f8ea94caf0284a0ef9638b76a82 in 1253ms, sequenceid=95, compaction requested=false 2024-11-10T19:18:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:22,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T19:18:22,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:22,157 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T19:18:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:22,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/9b86586f40a54e0f94354cc07202f459 is 50, key is test_row_0/A:col10/1731266300778/Put/seqid=0 2024-11-10T19:18:22,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742308_1484 (size=9657) 2024-11-10T19:18:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:22,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T19:18:22,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266302378, max timestamp=1731266302389Num uniques 859; 2024-11-10T19:18:22,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:22,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266302378, max timestamp=1731266302389Num uniques 859; 2024-11-10T19:18:22,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266302378, max timestamp=1731266302389Num uniques 859; 2024-11-10T19:18:22,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:22,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:22,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266362447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:22,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266362452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:22,569 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/9b86586f40a54e0f94354cc07202f459 2024-11-10T19:18:22,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266362561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:22,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/abda12b7ce0744e5a0abfd9dcfc79089 is 50, key is test_row_0/B:col10/1731266300778/Put/seqid=0 2024-11-10T19:18:22,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742309_1485 (size=9657) 2024-11-10T19:18:22,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266362571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:22,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266362782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:22,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266362791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:22,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:22,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266362961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:22,972 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in fef56f8ea94caf0284a0ef9638b76a82 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:22,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266362972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:22,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266362973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:22,978 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in fef56f8ea94caf0284a0ef9638b76a82 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:22,978 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/abda12b7ce0744e5a0abfd9dcfc79089 2024-11-10T19:18:22,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/dd3136f18d264b35981290a0c354a2f5 is 50, key is test_row_0/C:col10/1731266300778/Put/seqid=0 2024-11-10T19:18:22,986 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in fef56f8ea94caf0284a0ef9638b76a82 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:23,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742310_1486 (size=9657) 2024-11-10T19:18:23,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266363101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:23,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266363111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:23,421 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/dd3136f18d264b35981290a0c354a2f5 2024-11-10T19:18:23,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/9b86586f40a54e0f94354cc07202f459 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9b86586f40a54e0f94354cc07202f459 2024-11-10T19:18:23,442 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9b86586f40a54e0f94354cc07202f459, entries=100, sequenceid=107, filesize=9.4 K 2024-11-10T19:18:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/abda12b7ce0744e5a0abfd9dcfc79089 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/abda12b7ce0744e5a0abfd9dcfc79089 2024-11-10T19:18:23,457 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/abda12b7ce0744e5a0abfd9dcfc79089, entries=100, sequenceid=107, filesize=9.4 K 2024-11-10T19:18:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/dd3136f18d264b35981290a0c354a2f5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/dd3136f18d264b35981290a0c354a2f5 2024-11-10T19:18:23,472 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/dd3136f18d264b35981290a0c354a2f5, entries=100, sequenceid=107, filesize=9.4 K 2024-11-10T19:18:23,473 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fef56f8ea94caf0284a0ef9638b76a82 in 1316ms, sequenceid=107, compaction requested=true 2024-11-10T19:18:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-10T19:18:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-10T19:18:23,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-10T19:18:23,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3180 sec 2024-11-10T19:18:23,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.3270 sec 2024-11-10T19:18:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:23,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T19:18:23,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:23,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:23,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:23,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:23,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:23,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:23,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/6638991ec2ed4ec9b8f28a2b6a68dd92 is 50, key is test_row_0/A:col10/1731266303623/Put/seqid=0 2024-11-10T19:18:23,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742311_1487 (size=14541) 2024-11-10T19:18:23,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/6638991ec2ed4ec9b8f28a2b6a68dd92 2024-11-10T19:18:23,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266363673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:23,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266363686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:23,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:23,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/a8e577a033104038b8cb85a0adc3944a is 50, key is test_row_0/B:col10/1731266303623/Put/seqid=0 2024-11-10T19:18:23,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742312_1488 (size=12151) 2024-11-10T19:18:23,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:23,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266363801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266363801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:23,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:24,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266364021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:24,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266364021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:24,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/a8e577a033104038b8cb85a0adc3944a 2024-11-10T19:18:24,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/ebc47e5677d1495b9d7e1d206d8b8a7c is 50, key is test_row_0/C:col10/1731266303623/Put/seqid=0 2024-11-10T19:18:24,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742313_1489 (size=12151) 2024-11-10T19:18:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T19:18:24,341 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:24,342 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-10T19:18:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T19:18:24,343 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:24,344 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:24,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266364341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266364341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T19:18:24,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T19:18:24,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:24,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:24,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:24,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:24,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:24,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/ebc47e5677d1495b9d7e1d206d8b8a7c 2024-11-10T19:18:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/6638991ec2ed4ec9b8f28a2b6a68dd92 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/6638991ec2ed4ec9b8f28a2b6a68dd92 2024-11-10T19:18:24,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/6638991ec2ed4ec9b8f28a2b6a68dd92, entries=200, sequenceid=144, filesize=14.2 K 2024-11-10T19:18:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/a8e577a033104038b8cb85a0adc3944a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/a8e577a033104038b8cb85a0adc3944a 2024-11-10T19:18:24,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/a8e577a033104038b8cb85a0adc3944a, entries=150, sequenceid=144, filesize=11.9 K 2024-11-10T19:18:24,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/ebc47e5677d1495b9d7e1d206d8b8a7c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/ebc47e5677d1495b9d7e1d206d8b8a7c 2024-11-10T19:18:24,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/ebc47e5677d1495b9d7e1d206d8b8a7c, entries=150, sequenceid=144, filesize=11.9 K 2024-11-10T19:18:24,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for fef56f8ea94caf0284a0ef9638b76a82 in 923ms, sequenceid=144, compaction requested=true 2024-11-10T19:18:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:24,551 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:24,551 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:24,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:24,552 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50643 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:24,552 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/A is initiating minor compaction (all files) 2024-11-10T19:18:24,552 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:24,553 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/A in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:24,553 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/B is initiating minor compaction (all files) 2024-11-10T19:18:24,553 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/B in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:24,553 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb5ee0e9dbe54590abaf028a73b2bb7d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb8312285f2c400b8f4aa9251bf4ac8d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9b86586f40a54e0f94354cc07202f459, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/6638991ec2ed4ec9b8f28a2b6a68dd92] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=49.5 K 2024-11-10T19:18:24,553 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7640dcd8b25344b0a91f246b52b6542d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e85f96b27584589a8ac19ad0ec008f7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/abda12b7ce0744e5a0abfd9dcfc79089, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/a8e577a033104038b8cb85a0adc3944a] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=44.8 K 2024-11-10T19:18:24,553 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb5ee0e9dbe54590abaf028a73b2bb7d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731266299269 2024-11-10T19:18:24,553 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 7640dcd8b25344b0a91f246b52b6542d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731266299269 2024-11-10T19:18:24,553 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb8312285f2c400b8f4aa9251bf4ac8d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731266299557 2024-11-10T19:18:24,553 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e85f96b27584589a8ac19ad0ec008f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731266299557 2024-11-10T19:18:24,553 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b86586f40a54e0f94354cc07202f459, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731266300772 2024-11-10T19:18:24,553 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting abda12b7ce0744e5a0abfd9dcfc79089, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731266300772 2024-11-10T19:18:24,554 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e577a033104038b8cb85a0adc3944a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731266302441 2024-11-10T19:18:24,554 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6638991ec2ed4ec9b8f28a2b6a68dd92, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731266302441 2024-11-10T19:18:24,571 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:24,571 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/1926dd6f90834afaaab44530baff9573 is 50, key is test_row_0/A:col10/1731266303623/Put/seqid=0 2024-11-10T19:18:24,575 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:24,575 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/d12a63c08f234787bca8a00a5aacb0ac is 50, key is test_row_0/B:col10/1731266303623/Put/seqid=0 2024-11-10T19:18:24,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742315_1491 (size=12391) 2024-11-10T19:18:24,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742314_1490 (size=12391) 2024-11-10T19:18:24,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T19:18:24,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:24,649 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:18:24,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:24,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:24,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:24,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:24,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:24,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:24,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/32f249a15dfd4f5e9cf32002269f77a0 is 50, key is test_row_0/A:col10/1731266303669/Put/seqid=0 2024-11-10T19:18:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T19:18:24,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742316_1492 (size=9757) 2024-11-10T19:18:24,667 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/32f249a15dfd4f5e9cf32002269f77a0 2024-11-10T19:18:24,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/7b0ff4342e8a41f2a7b5490318d64bf5 is 50, key is test_row_0/B:col10/1731266303669/Put/seqid=0 2024-11-10T19:18:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742317_1493 (size=9757) 2024-11-10T19:18:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:24,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:24,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266304912, max timestamp=1731266304964Num uniques 859; 2024-11-10T19:18:24,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266304912, max timestamp=1731266304964Num uniques 859; 2024-11-10T19:18:24,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:24,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:24,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266304912, max timestamp=1731266304964Num uniques 859; 2024-11-10T19:18:24,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T19:18:24,995 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/d12a63c08f234787bca8a00a5aacb0ac as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d12a63c08f234787bca8a00a5aacb0ac 2024-11-10T19:18:25,000 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/1926dd6f90834afaaab44530baff9573 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/1926dd6f90834afaaab44530baff9573 2024-11-10T19:18:25,002 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/B of fef56f8ea94caf0284a0ef9638b76a82 into d12a63c08f234787bca8a00a5aacb0ac(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:25,002 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:25,002 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/B, priority=12, startTime=1731266304551; duration=0sec 2024-11-10T19:18:25,002 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:25,002 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:B 2024-11-10T19:18:25,002 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:25,003 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:25,004 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/C is initiating minor compaction (all files) 2024-11-10T19:18:25,004 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/C in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:25,004 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2416ef3ca4e6421a93906998cd7db5e5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/18da0a423f6749b4a91010cbd95acd7a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/dd3136f18d264b35981290a0c354a2f5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/ebc47e5677d1495b9d7e1d206d8b8a7c] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=44.8 K 2024-11-10T19:18:25,004 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 2416ef3ca4e6421a93906998cd7db5e5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731266299269 2024-11-10T19:18:25,005 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 18da0a423f6749b4a91010cbd95acd7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731266299557 2024-11-10T19:18:25,005 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting dd3136f18d264b35981290a0c354a2f5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731266300772 2024-11-10T19:18:25,005 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ebc47e5677d1495b9d7e1d206d8b8a7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731266302441 2024-11-10T19:18:25,009 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/A of fef56f8ea94caf0284a0ef9638b76a82 into 1926dd6f90834afaaab44530baff9573(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:25,009 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:25,009 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/A, priority=12, startTime=1731266304551; duration=0sec 2024-11-10T19:18:25,009 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:25,009 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:A 2024-11-10T19:18:25,016 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:25,017 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/b9a1658f23bb427792d05228f0540779 is 50, key is test_row_0/C:col10/1731266303623/Put/seqid=0 2024-11-10T19:18:25,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742318_1494 (size=12391) 2024-11-10T19:18:25,031 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/b9a1658f23bb427792d05228f0540779 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/b9a1658f23bb427792d05228f0540779 2024-11-10T19:18:25,036 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/C of fef56f8ea94caf0284a0ef9638b76a82 into b9a1658f23bb427792d05228f0540779(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:25,036 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:25,036 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/C, priority=12, startTime=1731266304551; duration=0sec 2024-11-10T19:18:25,037 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:25,037 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:C 2024-11-10T19:18:25,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266365042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266365043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731266305027, max timestamp=1731266305067Num uniques 1018; 2024-11-10T19:18:25,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T19:18:25,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731266305027, max timestamp=1731266305067Num uniques 1018; 2024-11-10T19:18:25,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T19:18:25,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731266305027, max timestamp=1731266305067Num uniques 1018; 2024-11-10T19:18:25,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T19:18:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,081 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/7b0ff4342e8a41f2a7b5490318d64bf5 2024-11-10T19:18:25,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/1d65fc0c131245c688a373e833a32955 is 50, key is test_row_0/C:col10/1731266303669/Put/seqid=0 2024-11-10T19:18:25,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742319_1495 (size=9757) 2024-11-10T19:18:25,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266365122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266365124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266365127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266365181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266365181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266365231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266365241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266365241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266365393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266365395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266365441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266365451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266365451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T19:18:25,492 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/1d65fc0c131245c688a373e833a32955 2024-11-10T19:18:25,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/32f249a15dfd4f5e9cf32002269f77a0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/32f249a15dfd4f5e9cf32002269f77a0 2024-11-10T19:18:25,504 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/32f249a15dfd4f5e9cf32002269f77a0, entries=100, sequenceid=154, filesize=9.5 K 2024-11-10T19:18:25,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/7b0ff4342e8a41f2a7b5490318d64bf5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7b0ff4342e8a41f2a7b5490318d64bf5 2024-11-10T19:18:25,511 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7b0ff4342e8a41f2a7b5490318d64bf5, entries=100, sequenceid=154, filesize=9.5 K 2024-11-10T19:18:25,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/1d65fc0c131245c688a373e833a32955 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1d65fc0c131245c688a373e833a32955 2024-11-10T19:18:25,514 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1d65fc0c131245c688a373e833a32955, entries=100, sequenceid=154, filesize=9.5 K 2024-11-10T19:18:25,515 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for fef56f8ea94caf0284a0ef9638b76a82 in 867ms, sequenceid=154, compaction requested=false 2024-11-10T19:18:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-10T19:18:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-10T19:18:25,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-10T19:18:25,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1720 sec 2024-11-10T19:18:25,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.1760 sec 2024-11-10T19:18:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:25,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=143.13 KB heapSize=404.45 KB 2024-11-10T19:18:25,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:25,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:25,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:25,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:25,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:25,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:25,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/e0040acb4a2e4ff981214e6370eb9fec is 50, key is test_row_0/A:col10/1731266305710/Put/seqid=0 2024-11-10T19:18:25,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742320_1496 (size=14541) 2024-11-10T19:18:25,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/e0040acb4a2e4ff981214e6370eb9fec 2024-11-10T19:18:25,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/1916a1dcb34941bea27c8b3203551de0 is 50, key is test_row_0/B:col10/1731266305710/Put/seqid=0 2024-11-10T19:18:25,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266365752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742321_1497 (size=12151) 2024-11-10T19:18:25,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266365752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266365754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/1916a1dcb34941bea27c8b3203551de0 2024-11-10T19:18:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266365762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266365762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:25,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/00b4bdbf815048e78e8b19bacb5ccaac is 50, key is test_row_0/C:col10/1731266305710/Put/seqid=0 2024-11-10T19:18:25,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742322_1498 (size=12151) 2024-11-10T19:18:25,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/00b4bdbf815048e78e8b19bacb5ccaac 2024-11-10T19:18:25,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/e0040acb4a2e4ff981214e6370eb9fec as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e0040acb4a2e4ff981214e6370eb9fec 2024-11-10T19:18:25,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e0040acb4a2e4ff981214e6370eb9fec, entries=200, sequenceid=211, filesize=14.2 K 2024-11-10T19:18:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/1916a1dcb34941bea27c8b3203551de0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/1916a1dcb34941bea27c8b3203551de0 2024-11-10T19:18:25,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/1916a1dcb34941bea27c8b3203551de0, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T19:18:25,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/00b4bdbf815048e78e8b19bacb5ccaac as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/00b4bdbf815048e78e8b19bacb5ccaac 2024-11-10T19:18:25,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/00b4bdbf815048e78e8b19bacb5ccaac, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T19:18:25,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~404.41 KB/414112, currentSize=40.25 KB/41220 for fef56f8ea94caf0284a0ef9638b76a82 in 89ms, sequenceid=211, compaction requested=true 2024-11-10T19:18:25,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:25,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:25,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:25,801 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:25,801 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:25,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:25,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:25,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:25,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:25,802 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:25,802 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:25,802 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/A is initiating minor compaction (all files) 2024-11-10T19:18:25,802 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/B is initiating minor compaction (all files) 2024-11-10T19:18:25,802 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/A in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:25,802 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/B in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:25,802 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/1926dd6f90834afaaab44530baff9573, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/32f249a15dfd4f5e9cf32002269f77a0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e0040acb4a2e4ff981214e6370eb9fec] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=35.8 K 2024-11-10T19:18:25,802 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d12a63c08f234787bca8a00a5aacb0ac, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7b0ff4342e8a41f2a7b5490318d64bf5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/1916a1dcb34941bea27c8b3203551de0] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=33.5 K 2024-11-10T19:18:25,802 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1926dd6f90834afaaab44530baff9573, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731266302441 2024-11-10T19:18:25,802 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting d12a63c08f234787bca8a00a5aacb0ac, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731266302441 2024-11-10T19:18:25,803 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f249a15dfd4f5e9cf32002269f77a0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731266303669 2024-11-10T19:18:25,803 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0ff4342e8a41f2a7b5490318d64bf5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731266303669 2024-11-10T19:18:25,803 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0040acb4a2e4ff981214e6370eb9fec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731266305099 2024-11-10T19:18:25,803 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 1916a1dcb34941bea27c8b3203551de0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731266305112 2024-11-10T19:18:25,814 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:25,814 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:25,814 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/92705a42b9cc435a9d68adba62dcbcea is 50, key is test_row_0/B:col10/1731266305710/Put/seqid=0 2024-11-10T19:18:25,814 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/ee47e507e7f94d83919a35ceec01842a is 50, key is test_row_0/A:col10/1731266305710/Put/seqid=0 2024-11-10T19:18:25,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742324_1500 (size=12493) 2024-11-10T19:18:25,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742323_1499 (size=12493) 2024-11-10T19:18:25,827 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/92705a42b9cc435a9d68adba62dcbcea as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/92705a42b9cc435a9d68adba62dcbcea 2024-11-10T19:18:25,831 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/B of fef56f8ea94caf0284a0ef9638b76a82 into 92705a42b9cc435a9d68adba62dcbcea(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:25,831 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:25,832 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/B, priority=13, startTime=1731266305801; duration=0sec 2024-11-10T19:18:25,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:25,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:B 2024-11-10T19:18:25,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:25,834 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:25,834 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/C is initiating minor compaction (all files) 2024-11-10T19:18:25,834 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/C in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:25,834 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/b9a1658f23bb427792d05228f0540779, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1d65fc0c131245c688a373e833a32955, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/00b4bdbf815048e78e8b19bacb5ccaac] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=33.5 K 2024-11-10T19:18:25,834 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a1658f23bb427792d05228f0540779, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731266302441 2024-11-10T19:18:25,834 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d65fc0c131245c688a373e833a32955, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731266303669 2024-11-10T19:18:25,835 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b4bdbf815048e78e8b19bacb5ccaac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731266305112 2024-11-10T19:18:25,848 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:25,848 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/80f080394a474b3b8c1926b684b19ed1 is 50, key is test_row_0/C:col10/1731266305710/Put/seqid=0 2024-11-10T19:18:25,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742325_1501 (size=12493) 2024-11-10T19:18:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:25,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T19:18:25,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:25,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:25,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:25,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:25,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:25,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:25,875 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/80f080394a474b3b8c1926b684b19ed1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/80f080394a474b3b8c1926b684b19ed1 2024-11-10T19:18:25,890 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/C of fef56f8ea94caf0284a0ef9638b76a82 into 80f080394a474b3b8c1926b684b19ed1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:25,890 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:25,890 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/C, priority=13, startTime=1731266305801; duration=0sec 2024-11-10T19:18:25,890 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:25,890 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:C 2024-11-10T19:18:25,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/9339cee9199d4cdaa6ce3c8328da5469 is 50, key is test_row_1/A:col10/1731266305751/Put/seqid=0 2024-11-10T19:18:25,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742326_1502 (size=9757) 2024-11-10T19:18:26,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266305961, max timestamp=1731266306004Num uniques 859; 2024-11-10T19:18:26,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:26,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266305961, max timestamp=1731266306004Num uniques 859; 2024-11-10T19:18:26,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:26,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266305961, max timestamp=1731266306004Num uniques 859; 2024-11-10T19:18:26,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:26,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266366108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266366114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,228 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/ee47e507e7f94d83919a35ceec01842a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ee47e507e7f94d83919a35ceec01842a 2024-11-10T19:18:26,233 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/A of fef56f8ea94caf0284a0ef9638b76a82 into ee47e507e7f94d83919a35ceec01842a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:26,233 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:26,233 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/A, priority=13, startTime=1731266305801; duration=0sec 2024-11-10T19:18:26,233 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:26,233 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:A 2024-11-10T19:18:26,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266366221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266366241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266366271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266366281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266366281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/9339cee9199d4cdaa6ce3c8328da5469 2024-11-10T19:18:26,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/27c043f9a3444183a040a11305e0e74c is 50, key is test_row_1/B:col10/1731266305751/Put/seqid=0 2024-11-10T19:18:26,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742327_1503 (size=9757) 2024-11-10T19:18:26,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266366441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266366461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T19:18:26,491 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:26,492 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-10T19:18:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T19:18:26,493 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:26,494 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:26,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T19:18:26,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T19:18:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:26,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:26,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/27c043f9a3444183a040a11305e0e74c 2024-11-10T19:18:26,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f1b577caeb2c4a92bf248d5c3f17ad97 is 50, key is test_row_1/C:col10/1731266305751/Put/seqid=0 2024-11-10T19:18:26,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742328_1504 (size=9757) 2024-11-10T19:18:26,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266366761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266366782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:26,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:26,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:26,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T19:18:26,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:26,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:26,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:26,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:26,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T19:18:26,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T19:18:26,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:26,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:26,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:26,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:26,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:27,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T19:18:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:27,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T19:18:27,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f1b577caeb2c4a92bf248d5c3f17ad97 2024-11-10T19:18:27,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/9339cee9199d4cdaa6ce3c8328da5469 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9339cee9199d4cdaa6ce3c8328da5469 2024-11-10T19:18:27,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9339cee9199d4cdaa6ce3c8328da5469, entries=100, sequenceid=222, filesize=9.5 K 2024-11-10T19:18:27,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/27c043f9a3444183a040a11305e0e74c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/27c043f9a3444183a040a11305e0e74c 2024-11-10T19:18:27,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/27c043f9a3444183a040a11305e0e74c, entries=100, sequenceid=222, filesize=9.5 K 2024-11-10T19:18:27,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f1b577caeb2c4a92bf248d5c3f17ad97 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f1b577caeb2c4a92bf248d5c3f17ad97 2024-11-10T19:18:27,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f1b577caeb2c4a92bf248d5c3f17ad97, entries=100, sequenceid=222, filesize=9.5 K 2024-11-10T19:18:27,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fef56f8ea94caf0284a0ef9638b76a82 in 1296ms, sequenceid=222, compaction requested=false 2024-11-10T19:18:27,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:27,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T19:18:27,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:27,260 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T19:18:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:27,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/908b6c452afb4553b5d68f67b7a5e687 is 50, key is test_row_0/A:col10/1731266306111/Put/seqid=0 2024-11-10T19:18:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:27,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:27,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742329_1505 (size=12251) 2024-11-10T19:18:27,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266367316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266367318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266367320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266367320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266367325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266367432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266367442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266367442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266367442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266367451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T19:18:27,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266367652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266367652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266367652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266367652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266367672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,710 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/908b6c452afb4553b5d68f67b7a5e687 2024-11-10T19:18:27,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/2e3e1638c9bd4a9abcdfdf2ec0c4ef0d is 50, key is test_row_0/B:col10/1731266306111/Put/seqid=0 2024-11-10T19:18:27,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742330_1506 (size=12251) 2024-11-10T19:18:27,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266367961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266367961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266367961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:27,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266367961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266367991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,142 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/2e3e1638c9bd4a9abcdfdf2ec0c4ef0d 2024-11-10T19:18:28,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/e422667b9cc64cbda371443ea7eab2c3 is 50, key is test_row_0/C:col10/1731266306111/Put/seqid=0 2024-11-10T19:18:28,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742331_1507 (size=12251) 2024-11-10T19:18:28,172 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/e422667b9cc64cbda371443ea7eab2c3 2024-11-10T19:18:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/908b6c452afb4553b5d68f67b7a5e687 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/908b6c452afb4553b5d68f67b7a5e687 2024-11-10T19:18:28,183 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/908b6c452afb4553b5d68f67b7a5e687, entries=150, sequenceid=261, filesize=12.0 K 2024-11-10T19:18:28,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/2e3e1638c9bd4a9abcdfdf2ec0c4ef0d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/2e3e1638c9bd4a9abcdfdf2ec0c4ef0d 2024-11-10T19:18:28,188 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/2e3e1638c9bd4a9abcdfdf2ec0c4ef0d, entries=150, sequenceid=261, filesize=12.0 K 2024-11-10T19:18:28,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/e422667b9cc64cbda371443ea7eab2c3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e422667b9cc64cbda371443ea7eab2c3 2024-11-10T19:18:28,194 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e422667b9cc64cbda371443ea7eab2c3, entries=150, sequenceid=261, filesize=12.0 K 2024-11-10T19:18:28,195 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for fef56f8ea94caf0284a0ef9638b76a82 in 935ms, sequenceid=261, compaction requested=true 2024-11-10T19:18:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-10T19:18:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-10T19:18:28,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-10T19:18:28,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.7060 sec 2024-11-10T19:18:28,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7020 sec 2024-11-10T19:18:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:28,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T19:18:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:28,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/bf6c7e275c854e9396e590a754901905 is 50, key is test_row_0/A:col10/1731266308486/Put/seqid=0 2024-11-10T19:18:28,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742332_1508 (size=17181) 2024-11-10T19:18:28,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266368565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266368568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266368580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266368582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266368582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T19:18:28,651 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:28,651 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-10T19:18:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T19:18:28,653 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:28,653 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:28,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:28,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266368691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266368691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266368691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266368702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266368702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T19:18:28,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T19:18:28,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:28,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:28,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:28,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:28,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:28,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266368911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266368911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266368911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266368921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:28,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266368921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:28,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:28,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/bf6c7e275c854e9396e590a754901905 2024-11-10T19:18:28,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/44720084ad79440a88081244591c7baf is 50, key is test_row_0/B:col10/1731266308486/Put/seqid=0 2024-11-10T19:18:28,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T19:18:28,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:28,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:28,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:28,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:28,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:28,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742333_1509 (size=12301) 2024-11-10T19:18:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T19:18:29,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T19:18:29,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:29,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266369221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266369231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266369231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266369241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266369241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T19:18:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T19:18:29,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/44720084ad79440a88081244591c7baf 2024-11-10T19:18:29,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/e38194b91faa4401b36c23c9c2f38601 is 50, key is test_row_0/C:col10/1731266308486/Put/seqid=0 2024-11-10T19:18:29,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742334_1510 (size=12301) 2024-11-10T19:18:29,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T19:18:29,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:29,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T19:18:29,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:29,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T19:18:29,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:29,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:29,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266369743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266369743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266369743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266369753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266369754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:29,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:29,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:29,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/e38194b91faa4401b36c23c9c2f38601 2024-11-10T19:18:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T19:18:29,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/bf6c7e275c854e9396e590a754901905 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/bf6c7e275c854e9396e590a754901905 2024-11-10T19:18:29,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/bf6c7e275c854e9396e590a754901905, entries=250, sequenceid=274, filesize=16.8 K 2024-11-10T19:18:29,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/44720084ad79440a88081244591c7baf as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/44720084ad79440a88081244591c7baf 2024-11-10T19:18:29,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/44720084ad79440a88081244591c7baf, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T19:18:29,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/e38194b91faa4401b36c23c9c2f38601 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e38194b91faa4401b36c23c9c2f38601 2024-11-10T19:18:29,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e38194b91faa4401b36c23c9c2f38601, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T19:18:29,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fef56f8ea94caf0284a0ef9638b76a82 in 1326ms, sequenceid=274, compaction requested=true 2024-11-10T19:18:29,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:29,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:29,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:29,812 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:29,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:29,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:29,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:29,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T19:18:29,814 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:29,814 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/A is initiating minor compaction (all files) 2024-11-10T19:18:29,814 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/A in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,814 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ee47e507e7f94d83919a35ceec01842a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9339cee9199d4cdaa6ce3c8328da5469, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/908b6c452afb4553b5d68f67b7a5e687, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/bf6c7e275c854e9396e590a754901905] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=50.5 K 2024-11-10T19:18:29,814 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee47e507e7f94d83919a35ceec01842a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731266305112 2024-11-10T19:18:29,816 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9339cee9199d4cdaa6ce3c8328da5469, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731266305751 2024-11-10T19:18:29,816 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 908b6c452afb4553b5d68f67b7a5e687, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731266306036 2024-11-10T19:18:29,817 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf6c7e275c854e9396e590a754901905, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731266307316 2024-11-10T19:18:29,822 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#A#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:29,822 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/010796bb6e944fd4a173eb26fd6c9a91 is 50, key is test_row_0/A:col10/1731266308486/Put/seqid=0 2024-11-10T19:18:29,824 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:29,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742335_1511 (size=12779) 2024-11-10T19:18:29,826 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:29,826 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/B is initiating minor compaction (all files) 2024-11-10T19:18:29,826 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/B in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,827 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/92705a42b9cc435a9d68adba62dcbcea, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/27c043f9a3444183a040a11305e0e74c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/2e3e1638c9bd4a9abcdfdf2ec0c4ef0d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/44720084ad79440a88081244591c7baf] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=45.7 K 2024-11-10T19:18:29,827 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 92705a42b9cc435a9d68adba62dcbcea, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731266305112 2024-11-10T19:18:29,828 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 27c043f9a3444183a040a11305e0e74c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731266305751 2024-11-10T19:18:29,828 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3e1638c9bd4a9abcdfdf2ec0c4ef0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731266306036 2024-11-10T19:18:29,828 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 44720084ad79440a88081244591c7baf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731266307317 2024-11-10T19:18:29,831 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/010796bb6e944fd4a173eb26fd6c9a91 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/010796bb6e944fd4a173eb26fd6c9a91 2024-11-10T19:18:29,835 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/A of fef56f8ea94caf0284a0ef9638b76a82 into 010796bb6e944fd4a173eb26fd6c9a91(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:29,835 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:29,835 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/A, priority=12, startTime=1731266309812; duration=0sec 2024-11-10T19:18:29,835 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:29,835 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:A 2024-11-10T19:18:29,835 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:29,837 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:29,837 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/C is initiating minor compaction (all files) 2024-11-10T19:18:29,837 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/C in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,837 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/80f080394a474b3b8c1926b684b19ed1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f1b577caeb2c4a92bf248d5c3f17ad97, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e422667b9cc64cbda371443ea7eab2c3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e38194b91faa4401b36c23c9c2f38601] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=45.7 K 2024-11-10T19:18:29,837 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f080394a474b3b8c1926b684b19ed1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731266305112 2024-11-10T19:18:29,838 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b577caeb2c4a92bf248d5c3f17ad97, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731266305751 2024-11-10T19:18:29,838 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting e422667b9cc64cbda371443ea7eab2c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731266306036 2024-11-10T19:18:29,838 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting e38194b91faa4401b36c23c9c2f38601, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731266307317 2024-11-10T19:18:29,855 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#B#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:29,855 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/905d00ac232e4e66a2384bcbac3b10e7 is 50, key is test_row_0/B:col10/1731266308486/Put/seqid=0 2024-11-10T19:18:29,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T19:18:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:29,874 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T19:18:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:29,876 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:29,877 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/3652d52711b7431da2a6a1ee671a30fc is 50, key is test_row_0/C:col10/1731266308486/Put/seqid=0 2024-11-10T19:18:29,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742336_1512 (size=12779) 2024-11-10T19:18:29,907 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/905d00ac232e4e66a2384bcbac3b10e7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/905d00ac232e4e66a2384bcbac3b10e7 2024-11-10T19:18:29,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/8045e771279f4d32ad958e4d04b008e7 is 50, key is test_row_0/A:col10/1731266308580/Put/seqid=0 2024-11-10T19:18:29,915 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/B of fef56f8ea94caf0284a0ef9638b76a82 into 905d00ac232e4e66a2384bcbac3b10e7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:29,915 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:29,915 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/B, priority=12, startTime=1731266309812; duration=0sec 2024-11-10T19:18:29,915 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:29,915 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:B 2024-11-10T19:18:29,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742338_1514 (size=12301) 2024-11-10T19:18:29,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742337_1513 (size=12779) 2024-11-10T19:18:29,993 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/3652d52711b7431da2a6a1ee671a30fc as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3652d52711b7431da2a6a1ee671a30fc 2024-11-10T19:18:30,001 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/C of fef56f8ea94caf0284a0ef9638b76a82 into 3652d52711b7431da2a6a1ee671a30fc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:30,002 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:30,002 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/C, priority=12, startTime=1731266309813; duration=0sec 2024-11-10T19:18:30,002 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:30,002 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:C 2024-11-10T19:18:30,362 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/8045e771279f4d32ad958e4d04b008e7 2024-11-10T19:18:30,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/459d2a70a66a41f0833885b39ee44a87 is 50, key is test_row_0/B:col10/1731266308580/Put/seqid=0 2024-11-10T19:18:30,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742339_1515 (size=12301) 2024-11-10T19:18:30,422 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/459d2a70a66a41f0833885b39ee44a87 2024-11-10T19:18:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/1e0b8aff2db244939b26c99cb5d7b815 is 50, key is test_row_0/C:col10/1731266308580/Put/seqid=0 2024-11-10T19:18:30,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742340_1516 (size=12301) 2024-11-10T19:18:30,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T19:18:30,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266370798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266370801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:30,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266370803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266370804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266370805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:30,878 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/1e0b8aff2db244939b26c99cb5d7b815 2024-11-10T19:18:30,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/8045e771279f4d32ad958e4d04b008e7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/8045e771279f4d32ad958e4d04b008e7 2024-11-10T19:18:30,887 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/8045e771279f4d32ad958e4d04b008e7, entries=150, sequenceid=294, filesize=12.0 K 2024-11-10T19:18:30,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/459d2a70a66a41f0833885b39ee44a87 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/459d2a70a66a41f0833885b39ee44a87 2024-11-10T19:18:30,891 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/459d2a70a66a41f0833885b39ee44a87, entries=150, sequenceid=294, filesize=12.0 K 2024-11-10T19:18:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/1e0b8aff2db244939b26c99cb5d7b815 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1e0b8aff2db244939b26c99cb5d7b815 2024-11-10T19:18:30,895 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1e0b8aff2db244939b26c99cb5d7b815, entries=150, sequenceid=294, filesize=12.0 K 2024-11-10T19:18:30,896 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fef56f8ea94caf0284a0ef9638b76a82 in 1022ms, sequenceid=294, compaction requested=false 2024-11-10T19:18:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-10T19:18:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-10T19:18:30,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-10T19:18:30,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2440 sec 2024-11-10T19:18:30,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.2490 sec 2024-11-10T19:18:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:30,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T19:18:30,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:30,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:30,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/ce98a4b2f9ed4628a2ea5e2ccef03e47 is 50, key is test_row_0/A:col10/1731266310790/Put/seqid=0 2024-11-10T19:18:30,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742341_1517 (size=14741) 2024-11-10T19:18:30,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/ce98a4b2f9ed4628a2ea5e2ccef03e47 2024-11-10T19:18:30,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/b65925dadbb44812b01e86dd6760563f is 50, key is test_row_0/B:col10/1731266310790/Put/seqid=0 2024-11-10T19:18:30,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742342_1518 (size=12301) 2024-11-10T19:18:30,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/b65925dadbb44812b01e86dd6760563f 2024-11-10T19:18:30,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/6db8084912a04d748c4590208bc0f68f is 50, key is test_row_0/C:col10/1731266310790/Put/seqid=0 2024-11-10T19:18:30,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742343_1519 (size=12301) 2024-11-10T19:18:30,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266370954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266370956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266370958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:30,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266370960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:30,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266370961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266371081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266371081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266371081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266371082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266371082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266371292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266371292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266371292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266371301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266371301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/6db8084912a04d748c4590208bc0f68f 2024-11-10T19:18:31,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/ce98a4b2f9ed4628a2ea5e2ccef03e47 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ce98a4b2f9ed4628a2ea5e2ccef03e47 2024-11-10T19:18:31,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ce98a4b2f9ed4628a2ea5e2ccef03e47, entries=200, sequenceid=310, filesize=14.4 K 2024-11-10T19:18:31,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/b65925dadbb44812b01e86dd6760563f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/b65925dadbb44812b01e86dd6760563f 2024-11-10T19:18:31,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/b65925dadbb44812b01e86dd6760563f, entries=150, sequenceid=310, filesize=12.0 K 2024-11-10T19:18:31,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/6db8084912a04d748c4590208bc0f68f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6db8084912a04d748c4590208bc0f68f 2024-11-10T19:18:31,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6db8084912a04d748c4590208bc0f68f, entries=150, sequenceid=310, filesize=12.0 K 2024-11-10T19:18:31,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fef56f8ea94caf0284a0ef9638b76a82 in 494ms, sequenceid=310, compaction requested=true 2024-11-10T19:18:31,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:31,421 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:31,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:31,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:31,421 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:31,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:31,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:31,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:31,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:31,422 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:31,422 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/A is initiating minor compaction (all files) 2024-11-10T19:18:31,422 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/A in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:31,422 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/010796bb6e944fd4a173eb26fd6c9a91, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/8045e771279f4d32ad958e4d04b008e7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ce98a4b2f9ed4628a2ea5e2ccef03e47] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=38.9 K 2024-11-10T19:18:31,422 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:31,422 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/B is initiating minor compaction (all files) 2024-11-10T19:18:31,423 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/B in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:31,423 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/905d00ac232e4e66a2384bcbac3b10e7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/459d2a70a66a41f0833885b39ee44a87, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/b65925dadbb44812b01e86dd6760563f] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=36.5 K 2024-11-10T19:18:31,423 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 010796bb6e944fd4a173eb26fd6c9a91, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731266307317 2024-11-10T19:18:31,423 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 905d00ac232e4e66a2384bcbac3b10e7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731266307317 2024-11-10T19:18:31,423 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8045e771279f4d32ad958e4d04b008e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731266308563 2024-11-10T19:18:31,423 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 459d2a70a66a41f0833885b39ee44a87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731266308563 2024-11-10T19:18:31,423 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce98a4b2f9ed4628a2ea5e2ccef03e47, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731266310790 2024-11-10T19:18:31,424 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting b65925dadbb44812b01e86dd6760563f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731266310790 2024-11-10T19:18:31,437 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:31,438 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/0c54cc2b7184420eb3200042cdc29c8e is 50, key is test_row_0/B:col10/1731266310790/Put/seqid=0 2024-11-10T19:18:31,446 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#A#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:31,447 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/4123867ca13547e7adb21c0f4e1ade53 is 50, key is test_row_0/A:col10/1731266310790/Put/seqid=0 2024-11-10T19:18:31,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742344_1520 (size=12881) 2024-11-10T19:18:31,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742345_1521 (size=12881) 2024-11-10T19:18:31,475 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/0c54cc2b7184420eb3200042cdc29c8e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/0c54cc2b7184420eb3200042cdc29c8e 2024-11-10T19:18:31,478 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/4123867ca13547e7adb21c0f4e1ade53 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4123867ca13547e7adb21c0f4e1ade53 2024-11-10T19:18:31,482 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/A of fef56f8ea94caf0284a0ef9638b76a82 into 4123867ca13547e7adb21c0f4e1ade53(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:31,482 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:31,482 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/A, priority=13, startTime=1731266311421; duration=0sec 2024-11-10T19:18:31,482 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:31,483 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:A 2024-11-10T19:18:31,483 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:31,485 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/B of fef56f8ea94caf0284a0ef9638b76a82 into 0c54cc2b7184420eb3200042cdc29c8e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:31,485 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:31,485 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/B, priority=13, startTime=1731266311421; duration=0sec 2024-11-10T19:18:31,485 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:31,485 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:B 2024-11-10T19:18:31,485 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:31,485 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/C is initiating minor compaction (all files) 2024-11-10T19:18:31,485 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/C in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:31,485 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3652d52711b7431da2a6a1ee671a30fc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1e0b8aff2db244939b26c99cb5d7b815, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6db8084912a04d748c4590208bc0f68f] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=36.5 K 2024-11-10T19:18:31,485 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3652d52711b7431da2a6a1ee671a30fc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731266307317 2024-11-10T19:18:31,486 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e0b8aff2db244939b26c99cb5d7b815, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731266308563 2024-11-10T19:18:31,487 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6db8084912a04d748c4590208bc0f68f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731266310790 2024-11-10T19:18:31,494 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:31,495 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/6704c0435b3e4d8c8a342dc1466551c8 is 50, key is test_row_0/C:col10/1731266310790/Put/seqid=0 2024-11-10T19:18:31,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742346_1522 (size=12881) 2024-11-10T19:18:31,519 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/6704c0435b3e4d8c8a342dc1466551c8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6704c0435b3e4d8c8a342dc1466551c8 2024-11-10T19:18:31,525 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/C of fef56f8ea94caf0284a0ef9638b76a82 into 6704c0435b3e4d8c8a342dc1466551c8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:31,525 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:31,525 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/C, priority=13, startTime=1731266311421; duration=0sec 2024-11-10T19:18:31,525 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:31,525 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:C 2024-11-10T19:18:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:31,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T19:18:31,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:31,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:31,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:31,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:31,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:31,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:31,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/b18ea0f747d64fba987cbb72f0d69ceb is 50, key is test_row_0/A:col10/1731266310960/Put/seqid=0 2024-11-10T19:18:31,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742347_1523 (size=14741) 2024-11-10T19:18:31,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/b18ea0f747d64fba987cbb72f0d69ceb 2024-11-10T19:18:31,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6302fb8e8cf44f1f884a56637be54daa is 50, key is test_row_0/B:col10/1731266310960/Put/seqid=0 2024-11-10T19:18:31,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266371633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266371634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266371642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266371645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266371645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742348_1524 (size=12301) 2024-11-10T19:18:31,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6302fb8e8cf44f1f884a56637be54daa 2024-11-10T19:18:31,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/489e357bfad4479a8cb53b1b224e29a2 is 50, key is test_row_0/C:col10/1731266310960/Put/seqid=0 2024-11-10T19:18:31,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742349_1525 (size=12301) 2024-11-10T19:18:31,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266371751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266371752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266371761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266371761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266371761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266371962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266371971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266371971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266371971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266371972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/489e357bfad4479a8cb53b1b224e29a2 2024-11-10T19:18:32,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/b18ea0f747d64fba987cbb72f0d69ceb as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b18ea0f747d64fba987cbb72f0d69ceb 2024-11-10T19:18:32,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b18ea0f747d64fba987cbb72f0d69ceb, entries=200, sequenceid=332, filesize=14.4 K 2024-11-10T19:18:32,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6302fb8e8cf44f1f884a56637be54daa as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6302fb8e8cf44f1f884a56637be54daa 2024-11-10T19:18:32,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6302fb8e8cf44f1f884a56637be54daa, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T19:18:32,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/489e357bfad4479a8cb53b1b224e29a2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/489e357bfad4479a8cb53b1b224e29a2 2024-11-10T19:18:32,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/489e357bfad4479a8cb53b1b224e29a2, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T19:18:32,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fef56f8ea94caf0284a0ef9638b76a82 in 538ms, sequenceid=332, compaction requested=false 2024-11-10T19:18:32,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:32,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T19:18:32,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:32,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:32,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:32,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:32,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:32,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:32,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/c0fe665cab06404497361be695d3700d is 50, key is test_row_0/A:col10/1731266311641/Put/seqid=0 2024-11-10T19:18:32,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742350_1526 (size=14741) 2024-11-10T19:18:32,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266372326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266372336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266372338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266372338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266372340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266372456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266372460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266372461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266372465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266372465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266372681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266372682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266372682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266372682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266372682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:32,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/c0fe665cab06404497361be695d3700d 2024-11-10T19:18:32,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/288988706c884574b159f162b7f70ab6 is 50, key is test_row_0/B:col10/1731266311641/Put/seqid=0 2024-11-10T19:18:32,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742351_1527 (size=12301) 2024-11-10T19:18:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T19:18:32,821 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:32,821 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-10T19:18:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T19:18:32,823 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:32,823 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:32,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T19:18:32,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T19:18:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:32,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:33,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266372991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266372991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266372991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266373001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266373001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T19:18:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:33,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:33,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:33,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:33,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:33,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/288988706c884574b159f162b7f70ab6 2024-11-10T19:18:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T19:18:33,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/7b52351a62054f0f9bb0ee99f7391113 is 50, key is test_row_0/C:col10/1731266311641/Put/seqid=0 2024-11-10T19:18:33,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742352_1528 (size=12301) 2024-11-10T19:18:33,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/7b52351a62054f0f9bb0ee99f7391113 2024-11-10T19:18:33,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/c0fe665cab06404497361be695d3700d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c0fe665cab06404497361be695d3700d 2024-11-10T19:18:33,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c0fe665cab06404497361be695d3700d, entries=200, sequenceid=346, filesize=14.4 K 2024-11-10T19:18:33,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/288988706c884574b159f162b7f70ab6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/288988706c884574b159f162b7f70ab6 2024-11-10T19:18:33,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/288988706c884574b159f162b7f70ab6, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T19:18:33,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/7b52351a62054f0f9bb0ee99f7391113 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/7b52351a62054f0f9bb0ee99f7391113 2024-11-10T19:18:33,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/7b52351a62054f0f9bb0ee99f7391113, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T19:18:33,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fef56f8ea94caf0284a0ef9638b76a82 in 890ms, sequenceid=346, compaction requested=true 2024-11-10T19:18:33,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:33,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:33,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:33,174 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:33,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:33,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:33,174 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:33,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:33,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:33,175 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:33,175 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/A is initiating minor compaction (all files) 2024-11-10T19:18:33,175 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/A in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:33,175 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4123867ca13547e7adb21c0f4e1ade53, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b18ea0f747d64fba987cbb72f0d69ceb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c0fe665cab06404497361be695d3700d] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=41.4 K 2024-11-10T19:18:33,175 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:33,175 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/B is initiating minor compaction (all files) 2024-11-10T19:18:33,175 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/B in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:33,175 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/0c54cc2b7184420eb3200042cdc29c8e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6302fb8e8cf44f1f884a56637be54daa, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/288988706c884574b159f162b7f70ab6] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=36.6 K 2024-11-10T19:18:33,176 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4123867ca13547e7adb21c0f4e1ade53, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731266310790 2024-11-10T19:18:33,176 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c54cc2b7184420eb3200042cdc29c8e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731266310790 2024-11-10T19:18:33,176 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting b18ea0f747d64fba987cbb72f0d69ceb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731266310956 2024-11-10T19:18:33,176 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6302fb8e8cf44f1f884a56637be54daa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731266310959 2024-11-10T19:18:33,176 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 288988706c884574b159f162b7f70ab6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731266311627 2024-11-10T19:18:33,176 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0fe665cab06404497361be695d3700d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731266311627 2024-11-10T19:18:33,190 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:33,191 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6129492b3051427c92315a2f55239c76 is 50, key is test_row_0/B:col10/1731266311641/Put/seqid=0 2024-11-10T19:18:33,194 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:33,194 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/84be9ee7e3324389a101ccb524344467 is 50, key is test_row_0/A:col10/1731266311641/Put/seqid=0 2024-11-10T19:18:33,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742353_1529 (size=12983) 2024-11-10T19:18:33,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742354_1530 (size=12983) 2024-11-10T19:18:33,228 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/84be9ee7e3324389a101ccb524344467 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/84be9ee7e3324389a101ccb524344467 2024-11-10T19:18:33,228 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/6129492b3051427c92315a2f55239c76 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6129492b3051427c92315a2f55239c76 2024-11-10T19:18:33,236 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/B of fef56f8ea94caf0284a0ef9638b76a82 into 6129492b3051427c92315a2f55239c76(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:33,236 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:33,236 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/B, priority=13, startTime=1731266313174; duration=0sec 2024-11-10T19:18:33,236 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/A of fef56f8ea94caf0284a0ef9638b76a82 into 84be9ee7e3324389a101ccb524344467(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:33,236 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:33,236 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/A, priority=13, startTime=1731266313174; duration=0sec 2024-11-10T19:18:33,236 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:33,236 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:A 2024-11-10T19:18:33,236 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:33,237 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:33,237 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:B 2024-11-10T19:18:33,240 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:33,240 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/C is initiating minor compaction (all files) 2024-11-10T19:18:33,240 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/C in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:33,240 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6704c0435b3e4d8c8a342dc1466551c8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/489e357bfad4479a8cb53b1b224e29a2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/7b52351a62054f0f9bb0ee99f7391113] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=36.6 K 2024-11-10T19:18:33,241 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6704c0435b3e4d8c8a342dc1466551c8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731266310790 2024-11-10T19:18:33,241 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 489e357bfad4479a8cb53b1b224e29a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731266310959 2024-11-10T19:18:33,241 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b52351a62054f0f9bb0ee99f7391113, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731266311627 2024-11-10T19:18:33,257 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:33,257 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f6387d860b4d4adbb5d8f0b5b9ea8b82 is 50, key is test_row_0/C:col10/1731266311641/Put/seqid=0 2024-11-10T19:18:33,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742355_1531 (size=12983) 2024-11-10T19:18:33,278 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f6387d860b4d4adbb5d8f0b5b9ea8b82 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f6387d860b4d4adbb5d8f0b5b9ea8b82 2024-11-10T19:18:33,281 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/C of fef56f8ea94caf0284a0ef9638b76a82 into f6387d860b4d4adbb5d8f0b5b9ea8b82(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:33,282 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:33,282 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/C, priority=13, startTime=1731266313174; duration=0sec 2024-11-10T19:18:33,282 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:33,282 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:C 2024-11-10T19:18:33,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T19:18:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:33,287 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T19:18:33,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:33,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:33,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:33,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:33,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:33,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:33,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/f627a1244087465199577fe193df1c46 is 50, key is test_row_0/A:col10/1731266312337/Put/seqid=0 2024-11-10T19:18:33,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742356_1532 (size=12301) 2024-11-10T19:18:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T19:18:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:33,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:33,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266373528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266373529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266373531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266373532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266373541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266373651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266373651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266373651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266373651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266373661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,713 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/f627a1244087465199577fe193df1c46 2024-11-10T19:18:33,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/50d3de99c3be4c369a99da10c539fd38 is 50, key is test_row_0/B:col10/1731266312337/Put/seqid=0 2024-11-10T19:18:33,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742357_1533 (size=12301) 2024-11-10T19:18:33,773 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/50d3de99c3be4c369a99da10c539fd38 2024-11-10T19:18:33,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/3db473ddfaed43f9acf852d1fc8e4bfe is 50, key is test_row_0/C:col10/1731266312337/Put/seqid=0 2024-11-10T19:18:33,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742358_1534 (size=12301) 2024-11-10T19:18:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266373861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266373861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266373861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266373861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266373881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T19:18:34,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266374181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266374181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266374181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266374181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,194 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/3db473ddfaed43f9acf852d1fc8e4bfe 2024-11-10T19:18:34,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/f627a1244087465199577fe193df1c46 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f627a1244087465199577fe193df1c46 2024-11-10T19:18:34,202 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f627a1244087465199577fe193df1c46, entries=150, sequenceid=367, filesize=12.0 K 2024-11-10T19:18:34,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/50d3de99c3be4c369a99da10c539fd38 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/50d3de99c3be4c369a99da10c539fd38 2024-11-10T19:18:34,206 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/50d3de99c3be4c369a99da10c539fd38, entries=150, sequenceid=367, filesize=12.0 K 2024-11-10T19:18:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/3db473ddfaed43f9acf852d1fc8e4bfe as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3db473ddfaed43f9acf852d1fc8e4bfe 2024-11-10T19:18:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266374201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,212 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3db473ddfaed43f9acf852d1fc8e4bfe, entries=150, sequenceid=367, filesize=12.0 K 2024-11-10T19:18:34,213 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fef56f8ea94caf0284a0ef9638b76a82 in 927ms, sequenceid=367, compaction requested=false 2024-11-10T19:18:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-10T19:18:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-10T19:18:34,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T19:18:34,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3920 sec 2024-11-10T19:18:34,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.3970 sec 2024-11-10T19:18:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:34,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T19:18:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:34,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:34,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/0bcbdaad1fac4f57b2c371432c6f8669 is 50, key is test_row_0/A:col10/1731266313538/Put/seqid=0 2024-11-10T19:18:34,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266374729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266374733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266374734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742359_1535 (size=14741) 2024-11-10T19:18:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/0bcbdaad1fac4f57b2c371432c6f8669 2024-11-10T19:18:34,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/3f1ae325b3294b9d938399bdc58f9cd3 is 50, key is test_row_0/B:col10/1731266313538/Put/seqid=0 2024-11-10T19:18:34,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266374745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266374747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742360_1536 (size=12301) 2024-11-10T19:18:34,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266374851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266374851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266374861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266374871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266374871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T19:18:34,971 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:34,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-10T19:18:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T19:18:34,973 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:34,974 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:34,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:35,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266375071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266375071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266375071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T19:18:35,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266375092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266375092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T19:18:35,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:35,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/3f1ae325b3294b9d938399bdc58f9cd3 2024-11-10T19:18:35,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/667f8d1962264b32a0747089bd0e0806 is 50, key is test_row_0/C:col10/1731266313538/Put/seqid=0 2024-11-10T19:18:35,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742361_1537 (size=12301) 2024-11-10T19:18:35,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T19:18:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T19:18:35,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266375392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266375392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266375392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266375411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266375411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T19:18:35,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:35,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T19:18:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T19:18:35,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/667f8d1962264b32a0747089bd0e0806 2024-11-10T19:18:35,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/0bcbdaad1fac4f57b2c371432c6f8669 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/0bcbdaad1fac4f57b2c371432c6f8669 2024-11-10T19:18:35,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/0bcbdaad1fac4f57b2c371432c6f8669, entries=200, sequenceid=382, filesize=14.4 K 2024-11-10T19:18:35,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/3f1ae325b3294b9d938399bdc58f9cd3 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/3f1ae325b3294b9d938399bdc58f9cd3 2024-11-10T19:18:35,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/3f1ae325b3294b9d938399bdc58f9cd3, entries=150, sequenceid=382, filesize=12.0 K 2024-11-10T19:18:35,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/667f8d1962264b32a0747089bd0e0806 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/667f8d1962264b32a0747089bd0e0806 2024-11-10T19:18:35,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/667f8d1962264b32a0747089bd0e0806, entries=150, sequenceid=382, filesize=12.0 K 2024-11-10T19:18:35,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fef56f8ea94caf0284a0ef9638b76a82 in 919ms, sequenceid=382, compaction requested=true 2024-11-10T19:18:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:35,613 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:35,613 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:35,614 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:35,614 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/A is initiating minor compaction (all files) 2024-11-10T19:18:35,614 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/A in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,614 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/84be9ee7e3324389a101ccb524344467, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f627a1244087465199577fe193df1c46, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/0bcbdaad1fac4f57b2c371432c6f8669] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=39.1 K 2024-11-10T19:18:35,614 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:35,614 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/B is initiating minor compaction (all files) 2024-11-10T19:18:35,614 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/B in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,614 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6129492b3051427c92315a2f55239c76, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/50d3de99c3be4c369a99da10c539fd38, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/3f1ae325b3294b9d938399bdc58f9cd3] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=36.7 K 2024-11-10T19:18:35,615 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6129492b3051427c92315a2f55239c76, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731266311627 2024-11-10T19:18:35,615 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84be9ee7e3324389a101ccb524344467, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731266311627 2024-11-10T19:18:35,615 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d3de99c3be4c369a99da10c539fd38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731266312312 2024-11-10T19:18:35,615 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f627a1244087465199577fe193df1c46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731266312312 2024-11-10T19:18:35,615 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f1ae325b3294b9d938399bdc58f9cd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731266313527 2024-11-10T19:18:35,615 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bcbdaad1fac4f57b2c371432c6f8669, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731266313516 2024-11-10T19:18:35,622 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#B#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:35,623 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#A#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:35,623 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/ee72e37c86474c4892d92dc784606256 is 50, key is test_row_0/B:col10/1731266313538/Put/seqid=0 2024-11-10T19:18:35,623 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/f2b5d2198c1f41e78cc236c3c10821bb is 50, key is test_row_0/A:col10/1731266313538/Put/seqid=0 2024-11-10T19:18:35,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742362_1538 (size=13085) 2024-11-10T19:18:35,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742363_1539 (size=13085) 2024-11-10T19:18:35,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T19:18:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:35,743 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:18:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:35,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/b46c34a2b503401ea2d3c68b69be97e8 is 50, key is test_row_0/A:col10/1731266314743/Put/seqid=0 2024-11-10T19:18:35,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742364_1540 (size=12301) 2024-11-10T19:18:35,898 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T19:18:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:35,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:35,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266375949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266375953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266375956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266375957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266375976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,041 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/ee72e37c86474c4892d92dc784606256 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/ee72e37c86474c4892d92dc784606256 2024-11-10T19:18:36,045 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/B of fef56f8ea94caf0284a0ef9638b76a82 into ee72e37c86474c4892d92dc784606256(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:36,045 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:36,046 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/B, priority=13, startTime=1731266315613; duration=0sec 2024-11-10T19:18:36,046 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:36,046 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:B 2024-11-10T19:18:36,046 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:36,049 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:36,049 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/C is initiating minor compaction (all files) 2024-11-10T19:18:36,049 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/f2b5d2198c1f41e78cc236c3c10821bb as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f2b5d2198c1f41e78cc236c3c10821bb 2024-11-10T19:18:36,049 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/C in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:36,049 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f6387d860b4d4adbb5d8f0b5b9ea8b82, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3db473ddfaed43f9acf852d1fc8e4bfe, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/667f8d1962264b32a0747089bd0e0806] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=36.7 K 2024-11-10T19:18:36,053 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting f6387d860b4d4adbb5d8f0b5b9ea8b82, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731266311627 2024-11-10T19:18:36,056 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db473ddfaed43f9acf852d1fc8e4bfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731266312312 2024-11-10T19:18:36,057 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 667f8d1962264b32a0747089bd0e0806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731266313527 2024-11-10T19:18:36,061 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/A of fef56f8ea94caf0284a0ef9638b76a82 into f2b5d2198c1f41e78cc236c3c10821bb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:36,061 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:36,061 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/A, priority=13, startTime=1731266315613; duration=0sec 2024-11-10T19:18:36,061 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:36,061 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:A 2024-11-10T19:18:36,066 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#C#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:36,066 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/2f57e9c4b0d848fcabee835e3957d868 is 50, key is test_row_0/C:col10/1731266313538/Put/seqid=0 2024-11-10T19:18:36,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742365_1541 (size=13085) 2024-11-10T19:18:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266376081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266376081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266376081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266376081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266376101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T19:18:36,154 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/b46c34a2b503401ea2d3c68b69be97e8 2024-11-10T19:18:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/9ca818e89d134131be6cb60c788d8500 is 50, key is test_row_0/B:col10/1731266314743/Put/seqid=0 2024-11-10T19:18:36,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742366_1542 (size=12301) 2024-11-10T19:18:36,169 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/9ca818e89d134131be6cb60c788d8500 2024-11-10T19:18:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/a41db0d59f584e2b93e8225dea78d83c is 50, key is test_row_0/C:col10/1731266314743/Put/seqid=0 2024-11-10T19:18:36,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742367_1543 (size=12301) 2024-11-10T19:18:36,182 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/a41db0d59f584e2b93e8225dea78d83c 2024-11-10T19:18:36,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/b46c34a2b503401ea2d3c68b69be97e8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b46c34a2b503401ea2d3c68b69be97e8 2024-11-10T19:18:36,188 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b46c34a2b503401ea2d3c68b69be97e8, entries=150, sequenceid=399, filesize=12.0 K 2024-11-10T19:18:36,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/9ca818e89d134131be6cb60c788d8500 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9ca818e89d134131be6cb60c788d8500 2024-11-10T19:18:36,194 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9ca818e89d134131be6cb60c788d8500, entries=150, sequenceid=399, filesize=12.0 K 2024-11-10T19:18:36,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/a41db0d59f584e2b93e8225dea78d83c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/a41db0d59f584e2b93e8225dea78d83c 2024-11-10T19:18:36,197 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/a41db0d59f584e2b93e8225dea78d83c, entries=150, sequenceid=399, filesize=12.0 K 2024-11-10T19:18:36,198 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fef56f8ea94caf0284a0ef9638b76a82 in 455ms, sequenceid=399, compaction requested=false 2024-11-10T19:18:36,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:36,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:36,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-10T19:18:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-10T19:18:36,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-10T19:18:36,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2250 sec 2024-11-10T19:18:36,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.2290 sec 2024-11-10T19:18:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:36,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:18:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:36,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/4b9d0c7165a7413d8e45608254e99393 is 50, key is test_row_0/A:col10/1731266316299/Put/seqid=0 2024-11-10T19:18:36,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742368_1544 (size=14741) 2024-11-10T19:18:36,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/4b9d0c7165a7413d8e45608254e99393 2024-11-10T19:18:36,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/9f3cd4b1d39946aa8139607d406a18f1 is 50, key is test_row_0/B:col10/1731266316299/Put/seqid=0 2024-11-10T19:18:36,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742369_1545 (size=12301) 2024-11-10T19:18:36,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/9f3cd4b1d39946aa8139607d406a18f1 2024-11-10T19:18:36,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/d3f248bc9b3e4ae3b19ffc5db55abd13 is 50, key is test_row_0/C:col10/1731266316299/Put/seqid=0 2024-11-10T19:18:36,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266376334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742370_1546 (size=12301) 2024-11-10T19:18:36,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266376335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266376337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266376341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266376342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266376451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266376462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266376462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266376462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266376462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,478 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/2f57e9c4b0d848fcabee835e3957d868 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2f57e9c4b0d848fcabee835e3957d868 2024-11-10T19:18:36,483 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/C of fef56f8ea94caf0284a0ef9638b76a82 into 2f57e9c4b0d848fcabee835e3957d868(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:36,483 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:36,483 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/C, priority=13, startTime=1731266315613; duration=0sec 2024-11-10T19:18:36,483 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:36,483 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:C 2024-11-10T19:18:36,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266376661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266376681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266376681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266376681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266376682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:36,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/d3f248bc9b3e4ae3b19ffc5db55abd13 2024-11-10T19:18:36,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/4b9d0c7165a7413d8e45608254e99393 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4b9d0c7165a7413d8e45608254e99393 2024-11-10T19:18:36,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4b9d0c7165a7413d8e45608254e99393, entries=200, sequenceid=417, filesize=14.4 K 2024-11-10T19:18:36,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/9f3cd4b1d39946aa8139607d406a18f1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9f3cd4b1d39946aa8139607d406a18f1 2024-11-10T19:18:36,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9f3cd4b1d39946aa8139607d406a18f1, entries=150, sequenceid=417, filesize=12.0 K 2024-11-10T19:18:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/d3f248bc9b3e4ae3b19ffc5db55abd13 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/d3f248bc9b3e4ae3b19ffc5db55abd13 2024-11-10T19:18:36,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/d3f248bc9b3e4ae3b19ffc5db55abd13, entries=150, sequenceid=417, filesize=12.0 K 2024-11-10T19:18:36,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fef56f8ea94caf0284a0ef9638b76a82 in 466ms, sequenceid=417, compaction requested=true 2024-11-10T19:18:36,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:36,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:36,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:36,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:36,766 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:36,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:36,767 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:36,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fef56f8ea94caf0284a0ef9638b76a82:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:36,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:36,767 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:36,767 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:36,767 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/A is initiating minor compaction (all files) 2024-11-10T19:18:36,767 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/B is initiating minor compaction (all files) 2024-11-10T19:18:36,767 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/A in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:36,767 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/B in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:36,767 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f2b5d2198c1f41e78cc236c3c10821bb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b46c34a2b503401ea2d3c68b69be97e8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4b9d0c7165a7413d8e45608254e99393] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=39.2 K 2024-11-10T19:18:36,767 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/ee72e37c86474c4892d92dc784606256, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9ca818e89d134131be6cb60c788d8500, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9f3cd4b1d39946aa8139607d406a18f1] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=36.8 K 2024-11-10T19:18:36,769 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b5d2198c1f41e78cc236c3c10821bb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731266313527 2024-11-10T19:18:36,769 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ee72e37c86474c4892d92dc784606256, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731266313527 2024-11-10T19:18:36,769 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting b46c34a2b503401ea2d3c68b69be97e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731266314714 2024-11-10T19:18:36,769 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca818e89d134131be6cb60c788d8500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731266314714 2024-11-10T19:18:36,769 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b9d0c7165a7413d8e45608254e99393, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731266315928 2024-11-10T19:18:36,769 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3cd4b1d39946aa8139607d406a18f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731266315948 2024-11-10T19:18:36,782 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#A#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:36,783 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/b6138ddaeda5442c9523c2f0526d7759 is 50, key is test_row_0/A:col10/1731266316299/Put/seqid=0 2024-11-10T19:18:36,790 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:36,791 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/4599c3cb2c064974a79f87871ad70767 is 50, key is test_row_0/B:col10/1731266316299/Put/seqid=0 2024-11-10T19:18:36,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742371_1547 (size=13187) 2024-11-10T19:18:36,812 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/b6138ddaeda5442c9523c2f0526d7759 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b6138ddaeda5442c9523c2f0526d7759 2024-11-10T19:18:36,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742372_1548 (size=13187) 2024-11-10T19:18:36,819 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/A of fef56f8ea94caf0284a0ef9638b76a82 into b6138ddaeda5442c9523c2f0526d7759(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:36,819 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:36,819 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/A, priority=13, startTime=1731266316766; duration=0sec 2024-11-10T19:18:36,819 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:36,819 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:A 2024-11-10T19:18:36,820 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:36,820 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:36,820 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): fef56f8ea94caf0284a0ef9638b76a82/C is initiating minor compaction (all files) 2024-11-10T19:18:36,820 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fef56f8ea94caf0284a0ef9638b76a82/C in TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:36,821 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2f57e9c4b0d848fcabee835e3957d868, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/a41db0d59f584e2b93e8225dea78d83c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/d3f248bc9b3e4ae3b19ffc5db55abd13] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp, totalSize=36.8 K 2024-11-10T19:18:36,821 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f57e9c4b0d848fcabee835e3957d868, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731266313527 2024-11-10T19:18:36,822 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting a41db0d59f584e2b93e8225dea78d83c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731266314714 2024-11-10T19:18:36,822 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f248bc9b3e4ae3b19ffc5db55abd13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731266315948 2024-11-10T19:18:36,833 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/4599c3cb2c064974a79f87871ad70767 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/4599c3cb2c064974a79f87871ad70767 2024-11-10T19:18:36,839 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/B of fef56f8ea94caf0284a0ef9638b76a82 into 4599c3cb2c064974a79f87871ad70767(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:36,839 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:36,840 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/B, priority=13, startTime=1731266316766; duration=0sec 2024-11-10T19:18:36,840 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:36,840 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:B 2024-11-10T19:18:36,842 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fef56f8ea94caf0284a0ef9638b76a82#C#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:36,843 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/ea99baa2e93148a18bb9b7d253afc34d is 50, key is test_row_0/C:col10/1731266316299/Put/seqid=0 2024-11-10T19:18:36,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742373_1549 (size=13187) 2024-11-10T19:18:36,871 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/ea99baa2e93148a18bb9b7d253afc34d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/ea99baa2e93148a18bb9b7d253afc34d 2024-11-10T19:18:36,876 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fef56f8ea94caf0284a0ef9638b76a82/C of fef56f8ea94caf0284a0ef9638b76a82 into ea99baa2e93148a18bb9b7d253afc34d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:36,876 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:36,876 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., storeName=fef56f8ea94caf0284a0ef9638b76a82/C, priority=13, startTime=1731266316767; duration=0sec 2024-11-10T19:18:36,876 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:36,876 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fef56f8ea94caf0284a0ef9638b76a82:C 2024-11-10T19:18:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:36,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:18:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:36,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/81c8474cd93c4d77a7e1853e64f2ebdf is 50, key is test_row_0/A:col10/1731266316324/Put/seqid=0 2024-11-10T19:18:36,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742374_1550 (size=14741) 2024-11-10T19:18:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266377009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266377010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266377011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266377012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266377013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T19:18:37,121 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:37,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-10T19:18:37,124 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T19:18:37,124 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:37,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266377131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266377131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266377131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266377132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266377132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T19:18:37,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T19:18:37,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:37,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266377341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266377353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266377353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266377353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266377353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/81c8474cd93c4d77a7e1853e64f2ebdf 2024-11-10T19:18:37,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/10a395359e8b488098e95fe0578dc48b is 50, key is test_row_0/B:col10/1731266316324/Put/seqid=0 2024-11-10T19:18:37,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742375_1551 (size=12301) 2024-11-10T19:18:37,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T19:18:37,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:37,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T19:18:37,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T19:18:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266377661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266377671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266377671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266377671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266377671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:37,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T19:18:37,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:37,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T19:18:37,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/10a395359e8b488098e95fe0578dc48b 2024-11-10T19:18:37,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/59b6866de27a4d038e703e63c75321f4 is 50, key is test_row_0/C:col10/1731266316324/Put/seqid=0 2024-11-10T19:18:37,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742376_1552 (size=12301) 2024-11-10T19:18:37,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T19:18:37,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:37,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:38,002 INFO [Thread-2105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2105. 2024-11-10T19:18:38,002 DEBUG [Thread-2105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:38,002 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,002 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:38,003 INFO [Thread-2103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2103. 2024-11-10T19:18:38,003 DEBUG [Thread-2103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:38,003 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,003 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:38,007 INFO [Thread-2107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2107. 2024-11-10T19:18:38,007 DEBUG [Thread-2107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:38,007 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,008 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:38,013 INFO [Thread-2109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2109. 2024-11-10T19:18:38,013 DEBUG [Thread-2109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:38,013 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,013 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:38,014 INFO [Thread-2101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2101. 2024-11-10T19:18:38,014 DEBUG [Thread-2101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:38,014 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,015 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:38,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:38,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T19:18:38,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:38,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:38,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:38,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46890 deadline: 1731266378181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46872 deadline: 1731266378181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46848 deadline: 1731266378181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46920 deadline: 1731266378182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46892 deadline: 1731266378182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 , the old value is region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., hostname=bdc25462c9ef,34487,1731266167729, seqNum=2 because the exception is null or not the one we care about 2024-11-10T19:18:38,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T19:18:38,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:38,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:38,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:38,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:38,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:38,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/59b6866de27a4d038e703e63c75321f4 2024-11-10T19:18:38,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/81c8474cd93c4d77a7e1853e64f2ebdf as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/81c8474cd93c4d77a7e1853e64f2ebdf 2024-11-10T19:18:38,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/81c8474cd93c4d77a7e1853e64f2ebdf, entries=200, sequenceid=438, filesize=14.4 K 2024-11-10T19:18:38,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/10a395359e8b488098e95fe0578dc48b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/10a395359e8b488098e95fe0578dc48b 2024-11-10T19:18:38,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/10a395359e8b488098e95fe0578dc48b, entries=150, sequenceid=438, filesize=12.0 K 2024-11-10T19:18:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/59b6866de27a4d038e703e63c75321f4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/59b6866de27a4d038e703e63c75321f4 2024-11-10T19:18:38,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/59b6866de27a4d038e703e63c75321f4, entries=150, sequenceid=438, filesize=12.0 K 2024-11-10T19:18:38,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fef56f8ea94caf0284a0ef9638b76a82 in 1259ms, sequenceid=438, compaction requested=false 2024-11-10T19:18:38,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T19:18:38,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T19:18:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:38,354 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T19:18:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:38,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/95ad2cb7af4341c7937448e135fc6f58 is 50, key is test_row_0/A:col10/1731266317012/Put/seqid=0 2024-11-10T19:18:38,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742377_1553 (size=12301) 2024-11-10T19:18:38,760 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/95ad2cb7af4341c7937448e135fc6f58 2024-11-10T19:18:38,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/47e2aecd00ee442b9b3e2af163ede8af is 50, key is test_row_0/B:col10/1731266317012/Put/seqid=0 2024-11-10T19:18:38,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742378_1554 (size=12301) 2024-11-10T19:18:39,169 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/47e2aecd00ee442b9b3e2af163ede8af 2024-11-10T19:18:39,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f10c29c28e95431a8ce5e0e15c6b32ea is 50, key is test_row_0/C:col10/1731266317012/Put/seqid=0 2024-11-10T19:18:39,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742379_1555 (size=12301) 2024-11-10T19:18:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. as already flushing 2024-11-10T19:18:39,192 INFO [Thread-2092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2092. 2024-11-10T19:18:39,192 DEBUG [Thread-2092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:39,192 INFO [Thread-2098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2098. 2024-11-10T19:18:39,193 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,193 INFO [Thread-2090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2090. 2024-11-10T19:18:39,193 DEBUG [Thread-2098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:39,193 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,193 DEBUG [Thread-2090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:39,193 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,193 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:39,193 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,193 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:39,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:39,194 INFO [Thread-2096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2096. 2024-11-10T19:18:39,194 DEBUG [Thread-2096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:39,194 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,194 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:39,202 INFO [Thread-2094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2094. 2024-11-10T19:18:39,202 DEBUG [Thread-2094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:18:39,202 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,203 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:39,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T19:18:39,578 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f10c29c28e95431a8ce5e0e15c6b32ea 2024-11-10T19:18:39,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/95ad2cb7af4341c7937448e135fc6f58 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/95ad2cb7af4341c7937448e135fc6f58 2024-11-10T19:18:39,583 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/95ad2cb7af4341c7937448e135fc6f58, entries=150, sequenceid=453, filesize=12.0 K 2024-11-10T19:18:39,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/47e2aecd00ee442b9b3e2af163ede8af as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/47e2aecd00ee442b9b3e2af163ede8af 2024-11-10T19:18:39,587 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/47e2aecd00ee442b9b3e2af163ede8af, entries=150, sequenceid=453, filesize=12.0 K 2024-11-10T19:18:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/f10c29c28e95431a8ce5e0e15c6b32ea as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f10c29c28e95431a8ce5e0e15c6b32ea 2024-11-10T19:18:39,590 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f10c29c28e95431a8ce5e0e15c6b32ea, entries=150, sequenceid=453, filesize=12.0 K 2024-11-10T19:18:39,591 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=33.54 KB/34350 for fef56f8ea94caf0284a0ef9638b76a82 in 1237ms, sequenceid=453, compaction requested=true 2024-11-10T19:18:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for fef56f8ea94caf0284a0ef9638b76a82: 2024-11-10T19:18:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-10T19:18:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-10T19:18:39,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-10T19:18:39,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4680 sec 2024-11-10T19:18:39,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.4720 sec 2024-11-10T19:18:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T19:18:41,281 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1016 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3047 rows 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1017 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3050 rows 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1020 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3057 rows 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1021 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3060 rows 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1019 2024-11-10T19:18:41,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-11-10T19:18:41,281 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:18:41,281 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:18:41,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:41,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:41,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:18:41,282 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T19:18:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T19:18:41,285 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266321285"}]},"ts":"1731266321285"} 2024-11-10T19:18:41,286 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T19:18:41,286 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T19:18:41,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T19:18:41,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fef56f8ea94caf0284a0ef9638b76a82, UNASSIGN}] 2024-11-10T19:18:41,288 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fef56f8ea94caf0284a0ef9638b76a82, UNASSIGN 2024-11-10T19:18:41,289 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=fef56f8ea94caf0284a0ef9638b76a82, regionState=CLOSING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:41,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fef56f8ea94caf0284a0ef9638b76a82, UNASSIGN because future has completed 2024-11-10T19:18:41,290 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T19:18:41,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:18:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T19:18:41,442 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(122): Close fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:41,442 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T19:18:41,442 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1722): Closing fef56f8ea94caf0284a0ef9638b76a82, disabling compactions & flushes 2024-11-10T19:18:41,442 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:41,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:41,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. after waiting 0 ms 2024-11-10T19:18:41,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:41,443 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(2902): Flushing fef56f8ea94caf0284a0ef9638b76a82 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T19:18:41,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=A 2024-11-10T19:18:41,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:41,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=B 2024-11-10T19:18:41,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:41,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fef56f8ea94caf0284a0ef9638b76a82, store=C 2024-11-10T19:18:41,443 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:41,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/e823b7444cef4d37a45464c8ec7ab4f2 is 50, key is test_row_0/A:col10/1731266319191/Put/seqid=0 2024-11-10T19:18:41,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742380_1556 (size=12301) 2024-11-10T19:18:41,452 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/e823b7444cef4d37a45464c8ec7ab4f2 2024-11-10T19:18:41,457 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/59555e3f4fc045bebc84d1b5af883450 is 50, key is test_row_0/B:col10/1731266319191/Put/seqid=0 2024-11-10T19:18:41,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742381_1557 (size=12301) 2024-11-10T19:18:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T19:18:41,865 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/59555e3f4fc045bebc84d1b5af883450 2024-11-10T19:18:41,871 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/c3b1f5ea3d7e4f989fc86450c376104d is 50, key is test_row_0/C:col10/1731266319191/Put/seqid=0 2024-11-10T19:18:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742382_1558 (size=12301) 2024-11-10T19:18:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T19:18:42,274 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/c3b1f5ea3d7e4f989fc86450c376104d 2024-11-10T19:18:42,277 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/A/e823b7444cef4d37a45464c8ec7ab4f2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e823b7444cef4d37a45464c8ec7ab4f2 2024-11-10T19:18:42,280 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e823b7444cef4d37a45464c8ec7ab4f2, entries=150, sequenceid=461, filesize=12.0 K 2024-11-10T19:18:42,281 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/B/59555e3f4fc045bebc84d1b5af883450 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/59555e3f4fc045bebc84d1b5af883450 2024-11-10T19:18:42,283 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/59555e3f4fc045bebc84d1b5af883450, entries=150, sequenceid=461, filesize=12.0 K 2024-11-10T19:18:42,284 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/.tmp/C/c3b1f5ea3d7e4f989fc86450c376104d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/c3b1f5ea3d7e4f989fc86450c376104d 2024-11-10T19:18:42,289 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/c3b1f5ea3d7e4f989fc86450c376104d, entries=150, sequenceid=461, filesize=12.0 K 2024-11-10T19:18:42,290 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for fef56f8ea94caf0284a0ef9638b76a82 in 847ms, sequenceid=461, compaction requested=true 2024-11-10T19:18:42,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c6472fb7039d49558c4061ed42e64975, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/72594e4bbafe48ca8714b22ac33d3ddc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/5c3cff7352c349cf979f26a7ab6df4a6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb5ee0e9dbe54590abaf028a73b2bb7d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb8312285f2c400b8f4aa9251bf4ac8d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9b86586f40a54e0f94354cc07202f459, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/6638991ec2ed4ec9b8f28a2b6a68dd92, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/1926dd6f90834afaaab44530baff9573, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/32f249a15dfd4f5e9cf32002269f77a0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e0040acb4a2e4ff981214e6370eb9fec, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ee47e507e7f94d83919a35ceec01842a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9339cee9199d4cdaa6ce3c8328da5469, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/908b6c452afb4553b5d68f67b7a5e687, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/bf6c7e275c854e9396e590a754901905, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/010796bb6e944fd4a173eb26fd6c9a91, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/8045e771279f4d32ad958e4d04b008e7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ce98a4b2f9ed4628a2ea5e2ccef03e47, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4123867ca13547e7adb21c0f4e1ade53, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b18ea0f747d64fba987cbb72f0d69ceb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c0fe665cab06404497361be695d3700d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/84be9ee7e3324389a101ccb524344467, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f627a1244087465199577fe193df1c46, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/0bcbdaad1fac4f57b2c371432c6f8669, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f2b5d2198c1f41e78cc236c3c10821bb, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b46c34a2b503401ea2d3c68b69be97e8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4b9d0c7165a7413d8e45608254e99393] to archive 2024-11-10T19:18:42,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:18:42,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c6472fb7039d49558c4061ed42e64975 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c6472fb7039d49558c4061ed42e64975 2024-11-10T19:18:42,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/72594e4bbafe48ca8714b22ac33d3ddc to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/72594e4bbafe48ca8714b22ac33d3ddc 2024-11-10T19:18:42,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/5c3cff7352c349cf979f26a7ab6df4a6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/5c3cff7352c349cf979f26a7ab6df4a6 2024-11-10T19:18:42,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb5ee0e9dbe54590abaf028a73b2bb7d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb5ee0e9dbe54590abaf028a73b2bb7d 2024-11-10T19:18:42,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb8312285f2c400b8f4aa9251bf4ac8d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/cb8312285f2c400b8f4aa9251bf4ac8d 2024-11-10T19:18:42,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9b86586f40a54e0f94354cc07202f459 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9b86586f40a54e0f94354cc07202f459 2024-11-10T19:18:42,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/6638991ec2ed4ec9b8f28a2b6a68dd92 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/6638991ec2ed4ec9b8f28a2b6a68dd92 2024-11-10T19:18:42,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/1926dd6f90834afaaab44530baff9573 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/1926dd6f90834afaaab44530baff9573 2024-11-10T19:18:42,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/32f249a15dfd4f5e9cf32002269f77a0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/32f249a15dfd4f5e9cf32002269f77a0 2024-11-10T19:18:42,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e0040acb4a2e4ff981214e6370eb9fec to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e0040acb4a2e4ff981214e6370eb9fec 2024-11-10T19:18:42,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ee47e507e7f94d83919a35ceec01842a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ee47e507e7f94d83919a35ceec01842a 2024-11-10T19:18:42,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9339cee9199d4cdaa6ce3c8328da5469 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/9339cee9199d4cdaa6ce3c8328da5469 2024-11-10T19:18:42,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/908b6c452afb4553b5d68f67b7a5e687 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/908b6c452afb4553b5d68f67b7a5e687 2024-11-10T19:18:42,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/bf6c7e275c854e9396e590a754901905 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/bf6c7e275c854e9396e590a754901905 2024-11-10T19:18:42,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/010796bb6e944fd4a173eb26fd6c9a91 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/010796bb6e944fd4a173eb26fd6c9a91 2024-11-10T19:18:42,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/8045e771279f4d32ad958e4d04b008e7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/8045e771279f4d32ad958e4d04b008e7 2024-11-10T19:18:42,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ce98a4b2f9ed4628a2ea5e2ccef03e47 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/ce98a4b2f9ed4628a2ea5e2ccef03e47 2024-11-10T19:18:42,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4123867ca13547e7adb21c0f4e1ade53 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4123867ca13547e7adb21c0f4e1ade53 2024-11-10T19:18:42,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b18ea0f747d64fba987cbb72f0d69ceb to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b18ea0f747d64fba987cbb72f0d69ceb 2024-11-10T19:18:42,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c0fe665cab06404497361be695d3700d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/c0fe665cab06404497361be695d3700d 2024-11-10T19:18:42,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/84be9ee7e3324389a101ccb524344467 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/84be9ee7e3324389a101ccb524344467 2024-11-10T19:18:42,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f627a1244087465199577fe193df1c46 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f627a1244087465199577fe193df1c46 2024-11-10T19:18:42,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/0bcbdaad1fac4f57b2c371432c6f8669 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/0bcbdaad1fac4f57b2c371432c6f8669 2024-11-10T19:18:42,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f2b5d2198c1f41e78cc236c3c10821bb to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/f2b5d2198c1f41e78cc236c3c10821bb 2024-11-10T19:18:42,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b46c34a2b503401ea2d3c68b69be97e8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b46c34a2b503401ea2d3c68b69be97e8 2024-11-10T19:18:42,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4b9d0c7165a7413d8e45608254e99393 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/4b9d0c7165a7413d8e45608254e99393 2024-11-10T19:18:42,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/c86d5cb044fa4417893552682ea8deb5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e9dab23089c44dda3969074f6f958d6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7640dcd8b25344b0a91f246b52b6542d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d8c16258c89c4294b4e606db5fb2b88d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e85f96b27584589a8ac19ad0ec008f7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/abda12b7ce0744e5a0abfd9dcfc79089, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d12a63c08f234787bca8a00a5aacb0ac, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/a8e577a033104038b8cb85a0adc3944a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7b0ff4342e8a41f2a7b5490318d64bf5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/92705a42b9cc435a9d68adba62dcbcea, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/1916a1dcb34941bea27c8b3203551de0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/27c043f9a3444183a040a11305e0e74c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/2e3e1638c9bd4a9abcdfdf2ec0c4ef0d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/905d00ac232e4e66a2384bcbac3b10e7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/44720084ad79440a88081244591c7baf, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/459d2a70a66a41f0833885b39ee44a87, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/0c54cc2b7184420eb3200042cdc29c8e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/b65925dadbb44812b01e86dd6760563f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6302fb8e8cf44f1f884a56637be54daa, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6129492b3051427c92315a2f55239c76, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/288988706c884574b159f162b7f70ab6, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/50d3de99c3be4c369a99da10c539fd38, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/ee72e37c86474c4892d92dc784606256, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/3f1ae325b3294b9d938399bdc58f9cd3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9ca818e89d134131be6cb60c788d8500, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9f3cd4b1d39946aa8139607d406a18f1] to archive 2024-11-10T19:18:42,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:18:42,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/c86d5cb044fa4417893552682ea8deb5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/c86d5cb044fa4417893552682ea8deb5 2024-11-10T19:18:42,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e9dab23089c44dda3969074f6f958d6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e9dab23089c44dda3969074f6f958d6 2024-11-10T19:18:42,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7640dcd8b25344b0a91f246b52b6542d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7640dcd8b25344b0a91f246b52b6542d 2024-11-10T19:18:42,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d8c16258c89c4294b4e606db5fb2b88d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d8c16258c89c4294b4e606db5fb2b88d 2024-11-10T19:18:42,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e85f96b27584589a8ac19ad0ec008f7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6e85f96b27584589a8ac19ad0ec008f7 2024-11-10T19:18:42,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/abda12b7ce0744e5a0abfd9dcfc79089 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/abda12b7ce0744e5a0abfd9dcfc79089 2024-11-10T19:18:42,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d12a63c08f234787bca8a00a5aacb0ac to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/d12a63c08f234787bca8a00a5aacb0ac 2024-11-10T19:18:42,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/a8e577a033104038b8cb85a0adc3944a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/a8e577a033104038b8cb85a0adc3944a 2024-11-10T19:18:42,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7b0ff4342e8a41f2a7b5490318d64bf5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/7b0ff4342e8a41f2a7b5490318d64bf5 2024-11-10T19:18:42,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/92705a42b9cc435a9d68adba62dcbcea to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/92705a42b9cc435a9d68adba62dcbcea 2024-11-10T19:18:42,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/1916a1dcb34941bea27c8b3203551de0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/1916a1dcb34941bea27c8b3203551de0 2024-11-10T19:18:42,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/27c043f9a3444183a040a11305e0e74c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/27c043f9a3444183a040a11305e0e74c 2024-11-10T19:18:42,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/2e3e1638c9bd4a9abcdfdf2ec0c4ef0d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/2e3e1638c9bd4a9abcdfdf2ec0c4ef0d 2024-11-10T19:18:42,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/905d00ac232e4e66a2384bcbac3b10e7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/905d00ac232e4e66a2384bcbac3b10e7 2024-11-10T19:18:42,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/44720084ad79440a88081244591c7baf to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/44720084ad79440a88081244591c7baf 2024-11-10T19:18:42,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/459d2a70a66a41f0833885b39ee44a87 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/459d2a70a66a41f0833885b39ee44a87 2024-11-10T19:18:42,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/0c54cc2b7184420eb3200042cdc29c8e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/0c54cc2b7184420eb3200042cdc29c8e 2024-11-10T19:18:42,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/b65925dadbb44812b01e86dd6760563f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/b65925dadbb44812b01e86dd6760563f 2024-11-10T19:18:42,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6302fb8e8cf44f1f884a56637be54daa to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6302fb8e8cf44f1f884a56637be54daa 2024-11-10T19:18:42,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6129492b3051427c92315a2f55239c76 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/6129492b3051427c92315a2f55239c76 2024-11-10T19:18:42,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/288988706c884574b159f162b7f70ab6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/288988706c884574b159f162b7f70ab6 2024-11-10T19:18:42,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/50d3de99c3be4c369a99da10c539fd38 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/50d3de99c3be4c369a99da10c539fd38 2024-11-10T19:18:42,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/ee72e37c86474c4892d92dc784606256 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/ee72e37c86474c4892d92dc784606256 2024-11-10T19:18:42,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/3f1ae325b3294b9d938399bdc58f9cd3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/3f1ae325b3294b9d938399bdc58f9cd3 2024-11-10T19:18:42,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9ca818e89d134131be6cb60c788d8500 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9ca818e89d134131be6cb60c788d8500 2024-11-10T19:18:42,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9f3cd4b1d39946aa8139607d406a18f1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/9f3cd4b1d39946aa8139607d406a18f1 2024-11-10T19:18:42,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f293edd8fe1144a8acdfa30ef4690071, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/39b5e5a227e3471087a30a46bb7ea8cc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2416ef3ca4e6421a93906998cd7db5e5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/bf17b9edb47c49638d2ce043b83acc34, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/18da0a423f6749b4a91010cbd95acd7a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/dd3136f18d264b35981290a0c354a2f5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/b9a1658f23bb427792d05228f0540779, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/ebc47e5677d1495b9d7e1d206d8b8a7c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1d65fc0c131245c688a373e833a32955, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/80f080394a474b3b8c1926b684b19ed1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/00b4bdbf815048e78e8b19bacb5ccaac, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f1b577caeb2c4a92bf248d5c3f17ad97, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e422667b9cc64cbda371443ea7eab2c3, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3652d52711b7431da2a6a1ee671a30fc, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e38194b91faa4401b36c23c9c2f38601, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1e0b8aff2db244939b26c99cb5d7b815, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6704c0435b3e4d8c8a342dc1466551c8, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6db8084912a04d748c4590208bc0f68f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/489e357bfad4479a8cb53b1b224e29a2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f6387d860b4d4adbb5d8f0b5b9ea8b82, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/7b52351a62054f0f9bb0ee99f7391113, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3db473ddfaed43f9acf852d1fc8e4bfe, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2f57e9c4b0d848fcabee835e3957d868, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/667f8d1962264b32a0747089bd0e0806, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/a41db0d59f584e2b93e8225dea78d83c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/d3f248bc9b3e4ae3b19ffc5db55abd13] to archive 2024-11-10T19:18:42,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:18:42,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f293edd8fe1144a8acdfa30ef4690071 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f293edd8fe1144a8acdfa30ef4690071 2024-11-10T19:18:42,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/39b5e5a227e3471087a30a46bb7ea8cc to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/39b5e5a227e3471087a30a46bb7ea8cc 2024-11-10T19:18:42,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2416ef3ca4e6421a93906998cd7db5e5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2416ef3ca4e6421a93906998cd7db5e5 2024-11-10T19:18:42,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/bf17b9edb47c49638d2ce043b83acc34 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/bf17b9edb47c49638d2ce043b83acc34 2024-11-10T19:18:42,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/18da0a423f6749b4a91010cbd95acd7a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/18da0a423f6749b4a91010cbd95acd7a 2024-11-10T19:18:42,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/dd3136f18d264b35981290a0c354a2f5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/dd3136f18d264b35981290a0c354a2f5 2024-11-10T19:18:42,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/b9a1658f23bb427792d05228f0540779 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/b9a1658f23bb427792d05228f0540779 2024-11-10T19:18:42,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/ebc47e5677d1495b9d7e1d206d8b8a7c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/ebc47e5677d1495b9d7e1d206d8b8a7c 2024-11-10T19:18:42,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1d65fc0c131245c688a373e833a32955 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1d65fc0c131245c688a373e833a32955 2024-11-10T19:18:42,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/80f080394a474b3b8c1926b684b19ed1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/80f080394a474b3b8c1926b684b19ed1 2024-11-10T19:18:42,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/00b4bdbf815048e78e8b19bacb5ccaac to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/00b4bdbf815048e78e8b19bacb5ccaac 2024-11-10T19:18:42,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f1b577caeb2c4a92bf248d5c3f17ad97 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f1b577caeb2c4a92bf248d5c3f17ad97 2024-11-10T19:18:42,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e422667b9cc64cbda371443ea7eab2c3 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e422667b9cc64cbda371443ea7eab2c3 2024-11-10T19:18:42,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3652d52711b7431da2a6a1ee671a30fc to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3652d52711b7431da2a6a1ee671a30fc 2024-11-10T19:18:42,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e38194b91faa4401b36c23c9c2f38601 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/e38194b91faa4401b36c23c9c2f38601 2024-11-10T19:18:42,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1e0b8aff2db244939b26c99cb5d7b815 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/1e0b8aff2db244939b26c99cb5d7b815 2024-11-10T19:18:42,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6704c0435b3e4d8c8a342dc1466551c8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6704c0435b3e4d8c8a342dc1466551c8 2024-11-10T19:18:42,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6db8084912a04d748c4590208bc0f68f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/6db8084912a04d748c4590208bc0f68f 2024-11-10T19:18:42,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/489e357bfad4479a8cb53b1b224e29a2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/489e357bfad4479a8cb53b1b224e29a2 2024-11-10T19:18:42,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f6387d860b4d4adbb5d8f0b5b9ea8b82 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f6387d860b4d4adbb5d8f0b5b9ea8b82 2024-11-10T19:18:42,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/7b52351a62054f0f9bb0ee99f7391113 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/7b52351a62054f0f9bb0ee99f7391113 2024-11-10T19:18:42,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3db473ddfaed43f9acf852d1fc8e4bfe to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/3db473ddfaed43f9acf852d1fc8e4bfe 2024-11-10T19:18:42,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2f57e9c4b0d848fcabee835e3957d868 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/2f57e9c4b0d848fcabee835e3957d868 2024-11-10T19:18:42,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/667f8d1962264b32a0747089bd0e0806 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/667f8d1962264b32a0747089bd0e0806 2024-11-10T19:18:42,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/a41db0d59f584e2b93e8225dea78d83c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/a41db0d59f584e2b93e8225dea78d83c 2024-11-10T19:18:42,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/d3f248bc9b3e4ae3b19ffc5db55abd13 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/d3f248bc9b3e4ae3b19ffc5db55abd13 2024-11-10T19:18:42,356 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/recovered.edits/464.seqid, newMaxSeqId=464, maxSeqId=1 2024-11-10T19:18:42,357 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. 2024-11-10T19:18:42,357 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1676): Region close journal for fef56f8ea94caf0284a0ef9638b76a82: Waiting for close lock at 1731266321442Running coprocessor pre-close hooks at 1731266321442Disabling compacts and flushes for region at 1731266321442Disabling writes for close at 1731266321443 (+1 ms)Obtaining lock to block concurrent updates at 1731266321443Preparing flush snapshotting stores in fef56f8ea94caf0284a0ef9638b76a82 at 1731266321443Finished memstore snapshotting TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731266321443Flushing stores of TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82. at 1731266321444 (+1 ms)Flushing fef56f8ea94caf0284a0ef9638b76a82/A: creating writer at 1731266321444Flushing fef56f8ea94caf0284a0ef9638b76a82/A: appending metadata at 1731266321446 (+2 ms)Flushing fef56f8ea94caf0284a0ef9638b76a82/A: closing flushed file at 1731266321446Flushing fef56f8ea94caf0284a0ef9638b76a82/B: creating writer at 1731266321455 (+9 ms)Flushing fef56f8ea94caf0284a0ef9638b76a82/B: appending metadata at 1731266321457 (+2 ms)Flushing fef56f8ea94caf0284a0ef9638b76a82/B: closing flushed file at 1731266321457Flushing fef56f8ea94caf0284a0ef9638b76a82/C: creating writer at 1731266321868 (+411 ms)Flushing fef56f8ea94caf0284a0ef9638b76a82/C: appending metadata at 1731266321870 (+2 ms)Flushing fef56f8ea94caf0284a0ef9638b76a82/C: closing flushed file at 1731266321870Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@394d25b0: reopening flushed file at 1731266322277 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b138963: reopening flushed file at 1731266322280 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76b53be7: reopening flushed file at 1731266322283 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for fef56f8ea94caf0284a0ef9638b76a82 in 847ms, sequenceid=461, compaction requested=true at 1731266322290 (+7 ms)Writing region close event to WAL at 1731266322354 (+64 ms)Running coprocessor post-close hooks at 1731266322357 (+3 ms)Closed at 1731266322357 2024-11-10T19:18:42,359 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(157): Closed fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:42,360 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=fef56f8ea94caf0284a0ef9638b76a82, regionState=CLOSED 2024-11-10T19:18:42,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:18:42,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-10T19:18:42,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseRegionProcedure fef56f8ea94caf0284a0ef9638b76a82, server=bdc25462c9ef,34487,1731266167729 in 1.0720 sec 2024-11-10T19:18:42,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-10T19:18:42,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fef56f8ea94caf0284a0ef9638b76a82, UNASSIGN in 1.0760 sec 2024-11-10T19:18:42,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T19:18:42,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0790 sec 2024-11-10T19:18:42,370 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266322369"}]},"ts":"1731266322369"} 2024-11-10T19:18:42,372 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T19:18:42,372 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T19:18:42,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0910 sec 2024-11-10T19:18:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T19:18:42,422 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:42,422 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T19:18:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:42,426 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T19:18:42,427 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=129, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:42,430 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:42,449 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/recovered.edits] 2024-11-10T19:18:42,454 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/81c8474cd93c4d77a7e1853e64f2ebdf to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/81c8474cd93c4d77a7e1853e64f2ebdf 2024-11-10T19:18:42,458 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/95ad2cb7af4341c7937448e135fc6f58 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/95ad2cb7af4341c7937448e135fc6f58 2024-11-10T19:18:42,459 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b6138ddaeda5442c9523c2f0526d7759 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/b6138ddaeda5442c9523c2f0526d7759 2024-11-10T19:18:42,461 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e823b7444cef4d37a45464c8ec7ab4f2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/A/e823b7444cef4d37a45464c8ec7ab4f2 2024-11-10T19:18:42,470 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/10a395359e8b488098e95fe0578dc48b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/10a395359e8b488098e95fe0578dc48b 2024-11-10T19:18:42,471 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/4599c3cb2c064974a79f87871ad70767 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/4599c3cb2c064974a79f87871ad70767 2024-11-10T19:18:42,473 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/47e2aecd00ee442b9b3e2af163ede8af to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/47e2aecd00ee442b9b3e2af163ede8af 2024-11-10T19:18:42,475 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/59555e3f4fc045bebc84d1b5af883450 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/B/59555e3f4fc045bebc84d1b5af883450 2024-11-10T19:18:42,477 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/59b6866de27a4d038e703e63c75321f4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/59b6866de27a4d038e703e63c75321f4 2024-11-10T19:18:42,478 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/c3b1f5ea3d7e4f989fc86450c376104d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/c3b1f5ea3d7e4f989fc86450c376104d 2024-11-10T19:18:42,479 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/ea99baa2e93148a18bb9b7d253afc34d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/ea99baa2e93148a18bb9b7d253afc34d 2024-11-10T19:18:42,480 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f10c29c28e95431a8ce5e0e15c6b32ea to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/C/f10c29c28e95431a8ce5e0e15c6b32ea 2024-11-10T19:18:42,483 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/recovered.edits/464.seqid to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82/recovered.edits/464.seqid 2024-11-10T19:18:42,483 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/fef56f8ea94caf0284a0ef9638b76a82 2024-11-10T19:18:42,484 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T19:18:42,487 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=129, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:42,490 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T19:18:42,493 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T19:18:42,494 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=129, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:42,494 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T19:18:42,495 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731266322494"}]},"ts":"9223372036854775807"} 2024-11-10T19:18:42,498 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T19:18:42,498 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fef56f8ea94caf0284a0ef9638b76a82, NAME => 'TestAcidGuarantees,,1731266295785.fef56f8ea94caf0284a0ef9638b76a82.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T19:18:42,498 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T19:18:42,499 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731266322498"}]},"ts":"9223372036854775807"} 2024-11-10T19:18:42,501 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T19:18:42,502 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=129, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:42,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-10T19:18:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T19:18:42,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T19:18:42,541 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:42,552 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=735 (was 805), ProcessCount=11 (was 11), AvailableMemoryMB=1413 (was 2177) 2024-11-10T19:18:42,574 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=735, ProcessCount=11, AvailableMemoryMB=1413 2024-11-10T19:18:42,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T19:18:42,576 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:18:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:42,578 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T19:18:42,578 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:42,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 130 2024-11-10T19:18:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T19:18:42,581 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T19:18:42,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742383_1559 (size=1061) 2024-11-10T19:18:42,633 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0 2024-11-10T19:18:42,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742384_1560 (size=53) 2024-11-10T19:18:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T19:18:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T19:18:43,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:18:43,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4988256ba260015146113518269b8182, disabling compactions & flushes 2024-11-10T19:18:43,063 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:43,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:43,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. after waiting 0 ms 2024-11-10T19:18:43,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:43,063 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:43,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4988256ba260015146113518269b8182: Waiting for close lock at 1731266323063Disabling compacts and flushes for region at 1731266323063Disabling writes for close at 1731266323063Writing region close event to WAL at 1731266323063Closed at 1731266323063 2024-11-10T19:18:43,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T19:18:43,065 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731266323064"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731266323064"}]},"ts":"1731266323064"} 2024-11-10T19:18:43,066 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T19:18:43,068 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T19:18:43,068 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266323068"}]},"ts":"1731266323068"} 2024-11-10T19:18:43,070 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T19:18:43,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, ASSIGN}] 2024-11-10T19:18:43,072 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, ASSIGN 2024-11-10T19:18:43,074 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, ASSIGN; state=OFFLINE, location=bdc25462c9ef,34487,1731266167729; forceNewPlan=false, retain=false 2024-11-10T19:18:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T19:18:43,224 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=4988256ba260015146113518269b8182, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:43,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, ASSIGN because future has completed 2024-11-10T19:18:43,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:18:43,379 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:43,379 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7752): Opening region: {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:18:43,380 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4988256ba260015146113518269b8182 2024-11-10T19:18:43,380 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:18:43,380 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7794): checking encryption for 4988256ba260015146113518269b8182 2024-11-10T19:18:43,380 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7797): checking classloading for 4988256ba260015146113518269b8182 2024-11-10T19:18:43,381 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4988256ba260015146113518269b8182 2024-11-10T19:18:43,382 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:18:43,382 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4988256ba260015146113518269b8182 columnFamilyName A 2024-11-10T19:18:43,382 DEBUG [StoreOpener-4988256ba260015146113518269b8182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:43,383 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(327): Store=4988256ba260015146113518269b8182/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:18:43,383 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4988256ba260015146113518269b8182 2024-11-10T19:18:43,384 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:18:43,384 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4988256ba260015146113518269b8182 columnFamilyName B 2024-11-10T19:18:43,384 DEBUG [StoreOpener-4988256ba260015146113518269b8182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:43,384 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(327): Store=4988256ba260015146113518269b8182/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:18:43,384 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4988256ba260015146113518269b8182 2024-11-10T19:18:43,385 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:18:43,385 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4988256ba260015146113518269b8182 columnFamilyName C 2024-11-10T19:18:43,385 DEBUG [StoreOpener-4988256ba260015146113518269b8182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:43,386 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(327): Store=4988256ba260015146113518269b8182/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:18:43,386 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:43,386 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1038): replaying wal for 4988256ba260015146113518269b8182 2024-11-10T19:18:43,386 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182 2024-11-10T19:18:43,387 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182 2024-11-10T19:18:43,387 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1048): stopping wal replay for 4988256ba260015146113518269b8182 2024-11-10T19:18:43,387 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1060): Cleaning up temporary data for 4988256ba260015146113518269b8182 2024-11-10T19:18:43,388 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:18:43,388 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): writing seq id for 4988256ba260015146113518269b8182 2024-11-10T19:18:43,390 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:18:43,390 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1114): Opened 4988256ba260015146113518269b8182; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71671596, jitterRate=0.0679900050163269}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:18:43,390 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4988256ba260015146113518269b8182 2024-11-10T19:18:43,390 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1006): Region open journal for 4988256ba260015146113518269b8182: Running coprocessor pre-open hook at 1731266323380Writing region info on filesystem at 1731266323380Initializing all the Stores at 1731266323380Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266323380Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266323381 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266323381Cleaning up temporary data from old regions at 1731266323387 (+6 ms)Running coprocessor post-open hooks at 1731266323390 (+3 ms)Region opened successfully at 1731266323390 2024-11-10T19:18:43,391 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., pid=132, masterSystemTime=1731266323377 2024-11-10T19:18:43,393 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:43,393 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:43,393 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=4988256ba260015146113518269b8182, regionState=OPEN, openSeqNum=2, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:43,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:18:43,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T19:18:43,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; OpenRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 in 169 msec 2024-11-10T19:18:43,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-10T19:18:43,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, ASSIGN in 326 msec 2024-11-10T19:18:43,398 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T19:18:43,399 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266323398"}]},"ts":"1731266323398"} 2024-11-10T19:18:43,400 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T19:18:43,400 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T19:18:43,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 824 msec 2024-11-10T19:18:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-10T19:18:43,721 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:43,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:43,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:43,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:43,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7deda434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:43,725 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c1ca40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:43,726 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:43,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:43,728 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T19:18:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is bdc25462c9ef,46761,1731266166843 2024-11-10T19:18:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e060b1d 2024-11-10T19:18:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T19:18:43,730 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T19:18:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T19:18:43,732 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:18:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T19:18:43,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742385_1561 (size=1097) 2024-11-10T19:18:44,145 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T19:18:44,145 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T19:18:44,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T19:18:44,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, REOPEN/MOVE}] 2024-11-10T19:18:44,149 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, REOPEN/MOVE 2024-11-10T19:18:44,150 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=4988256ba260015146113518269b8182, regionState=CLOSING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, REOPEN/MOVE because future has completed 2024-11-10T19:18:44,151 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T19:18:44,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:18:44,303 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 4988256ba260015146113518269b8182 2024-11-10T19:18:44,303 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T19:18:44,303 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 4988256ba260015146113518269b8182, disabling compactions & flushes 2024-11-10T19:18:44,303 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,303 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,303 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. after waiting 0 ms 2024-11-10T19:18:44,303 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,306 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T19:18:44,306 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,307 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 4988256ba260015146113518269b8182: Waiting for close lock at 1731266324303Running coprocessor pre-close hooks at 1731266324303Disabling compacts and flushes for region at 1731266324303Disabling writes for close at 1731266324303Writing region close event to WAL at 1731266324304 (+1 ms)Running coprocessor post-close hooks at 1731266324306 (+2 ms)Closed at 1731266324306 2024-11-10T19:18:44,307 WARN [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionServer(3299): Not adding moved region record: 4988256ba260015146113518269b8182 to self. 2024-11-10T19:18:44,308 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 4988256ba260015146113518269b8182 2024-11-10T19:18:44,308 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=4988256ba260015146113518269b8182, regionState=CLOSED 2024-11-10T19:18:44,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:18:44,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T19:18:44,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 in 159 msec 2024-11-10T19:18:44,313 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, REOPEN/MOVE; state=CLOSED, location=bdc25462c9ef,34487,1731266167729; forceNewPlan=false, retain=true 2024-11-10T19:18:44,463 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=4988256ba260015146113518269b8182, regionState=OPENING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, REOPEN/MOVE because future has completed 2024-11-10T19:18:44,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:18:44,618 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,619 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7752): Opening region: {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:18:44,619 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4988256ba260015146113518269b8182 2024-11-10T19:18:44,619 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:18:44,619 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7794): checking encryption for 4988256ba260015146113518269b8182 2024-11-10T19:18:44,619 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7797): checking classloading for 4988256ba260015146113518269b8182 2024-11-10T19:18:44,620 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4988256ba260015146113518269b8182 2024-11-10T19:18:44,620 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:18:44,621 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4988256ba260015146113518269b8182 columnFamilyName A 2024-11-10T19:18:44,621 DEBUG [StoreOpener-4988256ba260015146113518269b8182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:44,622 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(327): Store=4988256ba260015146113518269b8182/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:18:44,622 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4988256ba260015146113518269b8182 2024-11-10T19:18:44,622 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:18:44,622 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4988256ba260015146113518269b8182 columnFamilyName B 2024-11-10T19:18:44,622 DEBUG [StoreOpener-4988256ba260015146113518269b8182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:44,623 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(327): Store=4988256ba260015146113518269b8182/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:18:44,623 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4988256ba260015146113518269b8182 2024-11-10T19:18:44,623 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T19:18:44,623 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4988256ba260015146113518269b8182 columnFamilyName C 2024-11-10T19:18:44,623 DEBUG [StoreOpener-4988256ba260015146113518269b8182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:44,623 INFO [StoreOpener-4988256ba260015146113518269b8182-1 {}] regionserver.HStore(327): Store=4988256ba260015146113518269b8182/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:18:44,623 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,623 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1038): replaying wal for 4988256ba260015146113518269b8182 2024-11-10T19:18:44,624 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182 2024-11-10T19:18:44,624 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182 2024-11-10T19:18:44,625 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1048): stopping wal replay for 4988256ba260015146113518269b8182 2024-11-10T19:18:44,625 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1060): Cleaning up temporary data for 4988256ba260015146113518269b8182 2024-11-10T19:18:44,625 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T19:18:44,626 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1093): writing seq id for 4988256ba260015146113518269b8182 2024-11-10T19:18:44,626 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1114): Opened 4988256ba260015146113518269b8182; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72311824, jitterRate=0.0775301456451416}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T19:18:44,627 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4988256ba260015146113518269b8182 2024-11-10T19:18:44,627 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1006): Region open journal for 4988256ba260015146113518269b8182: Running coprocessor pre-open hook at 1731266324619Writing region info on filesystem at 1731266324619Initializing all the Stores at 1731266324620 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266324620Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266324620Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731266324620Cleaning up temporary data from old regions at 1731266324625 (+5 ms)Running coprocessor post-open hooks at 1731266324627 (+2 ms)Region opened successfully at 1731266324627 2024-11-10T19:18:44,628 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., pid=137, masterSystemTime=1731266324616 2024-11-10T19:18:44,629 DEBUG [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,629 INFO [RS_OPEN_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,629 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=4988256ba260015146113518269b8182, regionState=OPEN, openSeqNum=5, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:18:44,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=135 2024-11-10T19:18:44,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 in 166 msec 2024-11-10T19:18:44,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-10T19:18:44,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, REOPEN/MOVE in 483 msec 2024-11-10T19:18:44,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-10T19:18:44,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 487 msec 2024-11-10T19:18:44,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-10T19:18:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T19:18:44,639 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:44,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,640 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de0aaf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,641 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5024f9b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,643 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fed232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,645 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2711f31e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,647 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4af462, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,648 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ab4bbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,649 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec1d56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,650 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d3dcbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,652 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355d51ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,653 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e512c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,655 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4662a340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,657 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@488e1573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,659 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40714fcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,660 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e2994a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,662 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bcf28e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,663 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd441f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,665 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad03026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,666 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c10a00d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bdc25462c9ef,46761,-1 for getting cluster id 2024-11-10T19:18:44,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:18:44,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2805b0d-199f-46f8-9715-ae1ce58e3026' 2024-11-10T19:18:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:18:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2805b0d-199f-46f8-9715-ae1ce58e3026" 2024-11-10T19:18:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7627d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bdc25462c9ef,46761,-1] 2024-11-10T19:18:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:18:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:18:44,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:18:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d496a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:18:44,670 DEBUG [Thread-2505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,671 DEBUG [Thread-2507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,671 DEBUG [Thread-2498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,672 DEBUG [Thread-2496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,672 DEBUG [Thread-2492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,672 DEBUG [Thread-2490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,672 DEBUG [Thread-2494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,674 DEBUG [Thread-2501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,676 DEBUG [Thread-2503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,677 DEBUG [Thread-2509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:18:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bdc25462c9ef,34487,1731266167729, seqNum=-1] 2024-11-10T19:18:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:18:44,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:44,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:18:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-10T19:18:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T19:18:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,685 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:44,687 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:44,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5] 2024-11-10T19:18:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:44,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T19:18:44,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:44,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:44,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:44,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:44,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:44,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:44,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a508e36356274f64b1eed6b7c1b0bf00_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266324689/Put/seqid=0 2024-11-10T19:18:44,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742386_1562 (size=14594) 2024-11-10T19:18:44,731 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:44,734 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a508e36356274f64b1eed6b7c1b0bf00_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a508e36356274f64b1eed6b7c1b0bf00_4988256ba260015146113518269b8182 2024-11-10T19:18:44,735 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/ac2ec5d8abfa479a8d90b775946bc498, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:44,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/ac2ec5d8abfa479a8d90b775946bc498 is 175, key is test_row_0/A:col10/1731266324689/Put/seqid=0 2024-11-10T19:18:44,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742387_1563 (size=39549) 2024-11-10T19:18:44,757 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/ac2ec5d8abfa479a8d90b775946bc498 2024-11-10T19:18:44,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266384759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266384761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266384761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266384762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266384758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/52bfb3c9ffb34f4d8f63fff1ada55123 is 50, key is test_row_0/B:col10/1731266324689/Put/seqid=0 2024-11-10T19:18:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T19:18:44,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742388_1564 (size=12001) 2024-11-10T19:18:44,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T19:18:44,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:44,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:44,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266384872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266384872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266384872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266384872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266384873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:44,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T19:18:44,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:44,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:44,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:44,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T19:18:45,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266385081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266385081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266385081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266385081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266385082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T19:18:45,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:45,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:45,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:45,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/52bfb3c9ffb34f4d8f63fff1ada55123 2024-11-10T19:18:45,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/5e97338c23f54eaa9fa817bc4e5d6b4e is 50, key is test_row_0/C:col10/1731266324689/Put/seqid=0 2024-11-10T19:18:45,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T19:18:45,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:45,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:45,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:45,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T19:18:45,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742389_1565 (size=12001) 2024-11-10T19:18:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266385391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266385392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266385392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266385392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266385392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T19:18:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:45,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T19:18:45,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:45,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:45,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:45,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:45,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/5e97338c23f54eaa9fa817bc4e5d6b4e 2024-11-10T19:18:45,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/ac2ec5d8abfa479a8d90b775946bc498 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/ac2ec5d8abfa479a8d90b775946bc498 2024-11-10T19:18:45,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/ac2ec5d8abfa479a8d90b775946bc498, entries=200, sequenceid=16, filesize=38.6 K 2024-11-10T19:18:45,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/52bfb3c9ffb34f4d8f63fff1ada55123 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/52bfb3c9ffb34f4d8f63fff1ada55123 2024-11-10T19:18:45,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/52bfb3c9ffb34f4d8f63fff1ada55123, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T19:18:45,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/5e97338c23f54eaa9fa817bc4e5d6b4e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/5e97338c23f54eaa9fa817bc4e5d6b4e 2024-11-10T19:18:45,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/5e97338c23f54eaa9fa817bc4e5d6b4e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T19:18:45,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4988256ba260015146113518269b8182 in 1047ms, sequenceid=16, compaction requested=false 2024-11-10T19:18:45,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:45,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-10T19:18:45,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:45,772 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T19:18:45,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:45,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:45,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:45,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:45,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:45,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:45,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101f05813cc6ee49a9921105978bd48412_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266324757/Put/seqid=0 2024-11-10T19:18:45,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742390_1566 (size=12154) 2024-11-10T19:18:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T19:18:45,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266385906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266385910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266385910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266385912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266385912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266386021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266386021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266386022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266386022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266386022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:46,185 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101f05813cc6ee49a9921105978bd48412_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101f05813cc6ee49a9921105978bd48412_4988256ba260015146113518269b8182 2024-11-10T19:18:46,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7dea3822a18d42a683a3b8a4cc655e3a, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:46,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7dea3822a18d42a683a3b8a4cc655e3a is 175, key is test_row_0/A:col10/1731266324757/Put/seqid=0 2024-11-10T19:18:46,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742391_1567 (size=30955) 2024-11-10T19:18:46,189 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7dea3822a18d42a683a3b8a4cc655e3a 2024-11-10T19:18:46,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/a208690d97c24d13b3d450f8fe5866e9 is 50, key is test_row_0/B:col10/1731266324757/Put/seqid=0 2024-11-10T19:18:46,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742392_1568 (size=12001) 2024-11-10T19:18:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266386231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266386231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266386231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266386231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266386231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266386541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266386542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266386542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266386542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266386542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:46,597 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/a208690d97c24d13b3d450f8fe5866e9 2024-11-10T19:18:46,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/a6ddb71c05b34b8f9ac444f26a454e50 is 50, key is test_row_0/C:col10/1731266324757/Put/seqid=0 2024-11-10T19:18:46,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742393_1569 (size=12001) 2024-11-10T19:18:46,634 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/a6ddb71c05b34b8f9ac444f26a454e50 2024-11-10T19:18:46,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7dea3822a18d42a683a3b8a4cc655e3a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7dea3822a18d42a683a3b8a4cc655e3a 2024-11-10T19:18:46,644 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7dea3822a18d42a683a3b8a4cc655e3a, entries=150, sequenceid=36, filesize=30.2 K 2024-11-10T19:18:46,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/a208690d97c24d13b3d450f8fe5866e9 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a208690d97c24d13b3d450f8fe5866e9 2024-11-10T19:18:46,647 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a208690d97c24d13b3d450f8fe5866e9, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T19:18:46,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/a6ddb71c05b34b8f9ac444f26a454e50 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/a6ddb71c05b34b8f9ac444f26a454e50 2024-11-10T19:18:46,651 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/a6ddb71c05b34b8f9ac444f26a454e50, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T19:18:46,652 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4988256ba260015146113518269b8182 in 880ms, sequenceid=36, compaction requested=false 2024-11-10T19:18:46,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:46,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:46,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-10T19:18:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-10T19:18:46,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-10T19:18:46,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9670 sec 2024-11-10T19:18:46,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 1.9770 sec 2024-11-10T19:18:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T19:18:46,831 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:46,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-10T19:18:46,833 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T19:18:46,834 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:46,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T19:18:46,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-10T19:18:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:46,986 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T19:18:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111050e1c6163b714fb38ed3654ed361c6d9_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266325910/Put/seqid=0 2024-11-10T19:18:47,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742394_1570 (size=12154) 2024-11-10T19:18:47,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:47,029 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111050e1c6163b714fb38ed3654ed361c6d9_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111050e1c6163b714fb38ed3654ed361c6d9_4988256ba260015146113518269b8182 2024-11-10T19:18:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0e4c5e5257864113875e4a8d94ff1cfd, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0e4c5e5257864113875e4a8d94ff1cfd is 175, key is test_row_0/A:col10/1731266325910/Put/seqid=0 2024-11-10T19:18:47,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742395_1571 (size=30955) 2024-11-10T19:18:47,046 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0e4c5e5257864113875e4a8d94ff1cfd 2024-11-10T19:18:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:47,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:47,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/c9adff1f5d704c3498ea1c763540b695 is 50, key is test_row_0/B:col10/1731266325910/Put/seqid=0 2024-11-10T19:18:47,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742396_1572 (size=12001) 2024-11-10T19:18:47,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266387072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731266327067, max timestamp=1731266327077Num uniques 859; 2024-11-10T19:18:47,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T19:18:47,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266327067, max timestamp=1731266327077Num uniques 859; 2024-11-10T19:18:47,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731266327067, max timestamp=1731266327077Num uniques 859; 2024-11-10T19:18:47,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:47,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T19:18:47,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266387101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266387102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266387105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266387106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T19:18:47,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266387191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266387211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266387211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266387221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266387221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266387401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266387421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266387421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266387431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266387431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T19:18:47,462 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/c9adff1f5d704c3498ea1c763540b695 2024-11-10T19:18:47,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/302ef1c1b584486fb296453b06fc65a1 is 50, key is test_row_0/C:col10/1731266325910/Put/seqid=0 2024-11-10T19:18:47,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742397_1573 (size=12001) 2024-11-10T19:18:47,471 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/302ef1c1b584486fb296453b06fc65a1 2024-11-10T19:18:47,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0e4c5e5257864113875e4a8d94ff1cfd as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0e4c5e5257864113875e4a8d94ff1cfd 2024-11-10T19:18:47,482 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0e4c5e5257864113875e4a8d94ff1cfd, entries=150, sequenceid=48, filesize=30.2 K 2024-11-10T19:18:47,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/c9adff1f5d704c3498ea1c763540b695 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/c9adff1f5d704c3498ea1c763540b695 2024-11-10T19:18:47,487 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/c9adff1f5d704c3498ea1c763540b695, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T19:18:47,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/302ef1c1b584486fb296453b06fc65a1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/302ef1c1b584486fb296453b06fc65a1 2024-11-10T19:18:47,492 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/302ef1c1b584486fb296453b06fc65a1, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T19:18:47,493 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 4988256ba260015146113518269b8182 in 507ms, sequenceid=48, compaction requested=true 2024-11-10T19:18:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-10T19:18:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-10T19:18:47,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-10T19:18:47,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 661 msec 2024-11-10T19:18:47,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 666 msec 2024-11-10T19:18:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:47,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T19:18:47,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:47,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:47,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:47,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:47,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:47,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:47,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108434d7ce618b48e6914751e5d5670b16_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266327101/Put/seqid=0 2024-11-10T19:18:47,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742398_1574 (size=14594) 2024-11-10T19:18:47,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266387739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266387741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266387741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266387744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266387745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266387851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266387861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:47,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266387861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:47,877 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T19:18:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-10T19:18:47,971 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:47,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-10T19:18:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T19:18:47,973 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:47,974 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:47,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:48,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266388061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266388071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266388071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T19:18:48,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:48,144 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108434d7ce618b48e6914751e5d5670b16_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108434d7ce618b48e6914751e5d5670b16_4988256ba260015146113518269b8182 2024-11-10T19:18:48,145 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/483e3fab27bc4ab7814df63833a2d507, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:48,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/483e3fab27bc4ab7814df63833a2d507 is 175, key is test_row_0/A:col10/1731266327101/Put/seqid=0 2024-11-10T19:18:48,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742399_1575 (size=39549) 2024-11-10T19:18:48,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266388251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266388261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:48,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T19:18:48,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266388371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266388381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266388381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:48,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:48,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,550 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/483e3fab27bc4ab7814df63833a2d507 2024-11-10T19:18:48,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/60704bba4c9f486c9624960032c71b1a is 50, key is test_row_0/B:col10/1731266327101/Put/seqid=0 2024-11-10T19:18:48,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742400_1576 (size=12001) 2024-11-10T19:18:48,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:48,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:48,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T19:18:48,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:48,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:48,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266388881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:48,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:48,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:48,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266388891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266388892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:48,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/60704bba4c9f486c9624960032c71b1a 2024-11-10T19:18:48,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/7158588fd3e443ae8c33a5877cb2a536 is 50, key is test_row_0/C:col10/1731266327101/Put/seqid=0 2024-11-10T19:18:48,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742401_1577 (size=12001) 2024-11-10T19:18:49,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:49,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:49,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:49,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T19:18:49,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:49,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:49,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:49,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:49,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:49,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266389261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:49,263 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:49,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266389281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:49,285 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:49,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:49,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:49,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:49,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:49,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/7158588fd3e443ae8c33a5877cb2a536 2024-11-10T19:18:49,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/483e3fab27bc4ab7814df63833a2d507 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/483e3fab27bc4ab7814df63833a2d507 2024-11-10T19:18:49,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/483e3fab27bc4ab7814df63833a2d507, entries=200, sequenceid=83, filesize=38.6 K 2024-11-10T19:18:49,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/60704bba4c9f486c9624960032c71b1a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/60704bba4c9f486c9624960032c71b1a 2024-11-10T19:18:49,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/60704bba4c9f486c9624960032c71b1a, entries=150, sequenceid=83, filesize=11.7 K 2024-11-10T19:18:49,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/7158588fd3e443ae8c33a5877cb2a536 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/7158588fd3e443ae8c33a5877cb2a536 2024-11-10T19:18:49,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/7158588fd3e443ae8c33a5877cb2a536, entries=150, sequenceid=83, filesize=11.7 K 2024-11-10T19:18:49,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 4988256ba260015146113518269b8182 in 1677ms, sequenceid=83, compaction requested=true 2024-11-10T19:18:49,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:49,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:49,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:49,392 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:49,392 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:49,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:49,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:49,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:49,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:49,394 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:49,394 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:49,394 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/B is initiating minor compaction (all files) 2024-11-10T19:18:49,394 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/A is initiating minor compaction (all files) 2024-11-10T19:18:49,394 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/A in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,394 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/B in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,394 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/52bfb3c9ffb34f4d8f63fff1ada55123, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a208690d97c24d13b3d450f8fe5866e9, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/c9adff1f5d704c3498ea1c763540b695, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/60704bba4c9f486c9624960032c71b1a] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=46.9 K 2024-11-10T19:18:49,394 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/ac2ec5d8abfa479a8d90b775946bc498, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7dea3822a18d42a683a3b8a4cc655e3a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0e4c5e5257864113875e4a8d94ff1cfd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/483e3fab27bc4ab7814df63833a2d507] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=137.7 K 2024-11-10T19:18:49,394 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,394 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/ac2ec5d8abfa479a8d90b775946bc498, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7dea3822a18d42a683a3b8a4cc655e3a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0e4c5e5257864113875e4a8d94ff1cfd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/483e3fab27bc4ab7814df63833a2d507] 2024-11-10T19:18:49,394 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac2ec5d8abfa479a8d90b775946bc498, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731266324686 2024-11-10T19:18:49,394 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 52bfb3c9ffb34f4d8f63fff1ada55123, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731266324688 2024-11-10T19:18:49,395 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting a208690d97c24d13b3d450f8fe5866e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731266324750 2024-11-10T19:18:49,395 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dea3822a18d42a683a3b8a4cc655e3a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731266324750 2024-11-10T19:18:49,395 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e4c5e5257864113875e4a8d94ff1cfd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731266325904 2024-11-10T19:18:49,395 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c9adff1f5d704c3498ea1c763540b695, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731266325904 2024-11-10T19:18:49,395 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483e3fab27bc4ab7814df63833a2d507, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731266327094 2024-11-10T19:18:49,395 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 60704bba4c9f486c9624960032c71b1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731266327101 2024-11-10T19:18:49,403 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:49,405 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#B#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:49,406 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/b9fab8cde9bd474cb50a2f9a8fb87b08 is 50, key is test_row_0/B:col10/1731266327101/Put/seqid=0 2024-11-10T19:18:49,414 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111063999ae6c8da4d74a8e620b38b8b22e3_4988256ba260015146113518269b8182 store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:49,417 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111063999ae6c8da4d74a8e620b38b8b22e3_4988256ba260015146113518269b8182, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:49,417 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111063999ae6c8da4d74a8e620b38b8b22e3_4988256ba260015146113518269b8182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:49,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742402_1578 (size=12139) 2024-11-10T19:18:49,436 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/b9fab8cde9bd474cb50a2f9a8fb87b08 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b9fab8cde9bd474cb50a2f9a8fb87b08 2024-11-10T19:18:49,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742403_1579 (size=4469) 2024-11-10T19:18:49,441 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#A#compaction#487 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:49,442 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/998e5e5ba4d14569af2639b03b11f7ad is 175, key is test_row_0/A:col10/1731266327101/Put/seqid=0 2024-11-10T19:18:49,446 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/B of 4988256ba260015146113518269b8182 into b9fab8cde9bd474cb50a2f9a8fb87b08(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:49,447 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:49,447 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/B, priority=12, startTime=1731266329392; duration=0sec 2024-11-10T19:18:49,447 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:49,447 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:B 2024-11-10T19:18:49,447 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:49,448 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:49,448 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/C is initiating minor compaction (all files) 2024-11-10T19:18:49,448 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/C in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,448 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/5e97338c23f54eaa9fa817bc4e5d6b4e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/a6ddb71c05b34b8f9ac444f26a454e50, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/302ef1c1b584486fb296453b06fc65a1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/7158588fd3e443ae8c33a5877cb2a536] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=46.9 K 2024-11-10T19:18:49,448 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e97338c23f54eaa9fa817bc4e5d6b4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731266324688 2024-11-10T19:18:49,449 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting a6ddb71c05b34b8f9ac444f26a454e50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731266324750 2024-11-10T19:18:49,450 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 302ef1c1b584486fb296453b06fc65a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731266325904 2024-11-10T19:18:49,451 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 7158588fd3e443ae8c33a5877cb2a536, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731266327101 2024-11-10T19:18:49,459 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:49,459 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/06aea2bf80d34802846640a517fa8ed5 is 50, key is test_row_0/C:col10/1731266327101/Put/seqid=0 2024-11-10T19:18:49,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742404_1580 (size=31093) 2024-11-10T19:18:49,464 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/998e5e5ba4d14569af2639b03b11f7ad as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/998e5e5ba4d14569af2639b03b11f7ad 2024-11-10T19:18:49,468 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/A of 4988256ba260015146113518269b8182 into 998e5e5ba4d14569af2639b03b11f7ad(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:49,468 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:49,469 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/A, priority=12, startTime=1731266329392; duration=0sec 2024-11-10T19:18:49,469 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:49,469 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:A 2024-11-10T19:18:49,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742405_1581 (size=12139) 2024-11-10T19:18:49,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T19:18:49,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:49,503 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T19:18:49,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:49,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111051f777d449cb477bb6f908217452b09e_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266327737/Put/seqid=0 2024-11-10T19:18:49,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742406_1582 (size=12154) 2024-11-10T19:18:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:49,556 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111051f777d449cb477bb6f908217452b09e_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111051f777d449cb477bb6f908217452b09e_4988256ba260015146113518269b8182 2024-11-10T19:18:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7d473bacada84d09b86a11ce621c387f, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7d473bacada84d09b86a11ce621c387f is 175, key is test_row_0/A:col10/1731266327737/Put/seqid=0 2024-11-10T19:18:49,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742407_1583 (size=30955) 2024-11-10T19:18:49,875 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/06aea2bf80d34802846640a517fa8ed5 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/06aea2bf80d34802846640a517fa8ed5 2024-11-10T19:18:49,880 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/C of 4988256ba260015146113518269b8182 into 06aea2bf80d34802846640a517fa8ed5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:49,880 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:49,880 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/C, priority=12, startTime=1731266329392; duration=0sec 2024-11-10T19:18:49,880 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:49,880 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:C 2024-11-10T19:18:49,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:49,971 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7d473bacada84d09b86a11ce621c387f 2024-11-10T19:18:49,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731266329921, max timestamp=1731266329944Num uniques 859; 2024-11-10T19:18:49,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:18:49,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731266329921, max timestamp=1731266329944Num uniques 859; 2024-11-10T19:18:49,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:18:49,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731266329921, max timestamp=1731266329944Num uniques 859; 2024-11-10T19:18:49,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bdc25462c9ef:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T19:18:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/06234cdd19504f97b114da8abc86687f is 50, key is test_row_0/B:col10/1731266327737/Put/seqid=0 2024-11-10T19:18:49,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742408_1584 (size=12001) 2024-11-10T19:18:49,984 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/06234cdd19504f97b114da8abc86687f 2024-11-10T19:18:49,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/3082f6699dbe498bbafbd25384e41b5c is 50, key is test_row_0/C:col10/1731266327737/Put/seqid=0 2024-11-10T19:18:50,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742409_1585 (size=12001) 2024-11-10T19:18:50,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266390016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266390018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266390018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T19:18:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266390131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266390131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266390131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266390341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266390341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266390341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,406 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/3082f6699dbe498bbafbd25384e41b5c 2024-11-10T19:18:50,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7d473bacada84d09b86a11ce621c387f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d473bacada84d09b86a11ce621c387f 2024-11-10T19:18:50,415 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d473bacada84d09b86a11ce621c387f, entries=150, sequenceid=96, filesize=30.2 K 2024-11-10T19:18:50,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/06234cdd19504f97b114da8abc86687f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/06234cdd19504f97b114da8abc86687f 2024-11-10T19:18:50,418 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/06234cdd19504f97b114da8abc86687f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T19:18:50,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/3082f6699dbe498bbafbd25384e41b5c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/3082f6699dbe498bbafbd25384e41b5c 2024-11-10T19:18:50,421 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/3082f6699dbe498bbafbd25384e41b5c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T19:18:50,422 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4988256ba260015146113518269b8182 in 919ms, sequenceid=96, compaction requested=false 2024-11-10T19:18:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-10T19:18:50,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-10T19:18:50,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-10T19:18:50,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4490 sec 2024-11-10T19:18:50,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.4550 sec 2024-11-10T19:18:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:50,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T19:18:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T19:18:50,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111078471b70dae74c458ce4836b8a907892_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266330017/Put/seqid=0 2024-11-10T19:18:50,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742410_1586 (size=12304) 2024-11-10T19:18:50,670 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:50,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111078471b70dae74c458ce4836b8a907892_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111078471b70dae74c458ce4836b8a907892_4988256ba260015146113518269b8182 2024-11-10T19:18:50,673 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0f985502107744f48cf56ffd8e5efa63, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:50,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0f985502107744f48cf56ffd8e5efa63 is 175, key is test_row_0/A:col10/1731266330017/Put/seqid=0 2024-11-10T19:18:50,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266390671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266390675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266390675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742411_1587 (size=31105) 2024-11-10T19:18:50,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266390781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266390791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266390791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:50,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266390991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266391001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266391001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,141 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0f985502107744f48cf56ffd8e5efa63 2024-11-10T19:18:51,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/5dbcd9b9396b4653bf4f0fc879d15b93 is 50, key is test_row_0/B:col10/1731266330017/Put/seqid=0 2024-11-10T19:18:51,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742412_1588 (size=12151) 2024-11-10T19:18:51,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/5dbcd9b9396b4653bf4f0fc879d15b93 2024-11-10T19:18:51,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/92f1717d90544e9ab39e84950496861f is 50, key is test_row_0/C:col10/1731266330017/Put/seqid=0 2024-11-10T19:18:51,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742413_1589 (size=12151) 2024-11-10T19:18:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266391291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266391291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,297 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,298 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266391302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266391311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266391311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/92f1717d90544e9ab39e84950496861f 2024-11-10T19:18:51,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0f985502107744f48cf56ffd8e5efa63 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0f985502107744f48cf56ffd8e5efa63 2024-11-10T19:18:51,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0f985502107744f48cf56ffd8e5efa63, entries=150, sequenceid=133, filesize=30.4 K 2024-11-10T19:18:51,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/5dbcd9b9396b4653bf4f0fc879d15b93 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/5dbcd9b9396b4653bf4f0fc879d15b93 2024-11-10T19:18:51,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/5dbcd9b9396b4653bf4f0fc879d15b93, entries=150, sequenceid=133, filesize=11.9 K 2024-11-10T19:18:51,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/92f1717d90544e9ab39e84950496861f as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/92f1717d90544e9ab39e84950496861f 2024-11-10T19:18:51,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/92f1717d90544e9ab39e84950496861f, entries=150, sequenceid=133, filesize=11.9 K 2024-11-10T19:18:51,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 4988256ba260015146113518269b8182 in 958ms, sequenceid=133, compaction requested=true 2024-11-10T19:18:51,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:51,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:51,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:51,614 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:51,614 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:51,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:51,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:51,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:51,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:51,615 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93153 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:51,615 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:51,615 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/B is initiating minor compaction (all files) 2024-11-10T19:18:51,615 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/A is initiating minor compaction (all files) 2024-11-10T19:18:51,615 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/A in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:51,615 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/B in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:51,616 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b9fab8cde9bd474cb50a2f9a8fb87b08, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/06234cdd19504f97b114da8abc86687f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/5dbcd9b9396b4653bf4f0fc879d15b93] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=35.4 K 2024-11-10T19:18:51,616 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/998e5e5ba4d14569af2639b03b11f7ad, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d473bacada84d09b86a11ce621c387f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0f985502107744f48cf56ffd8e5efa63] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=91.0 K 2024-11-10T19:18:51,616 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:51,616 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/998e5e5ba4d14569af2639b03b11f7ad, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d473bacada84d09b86a11ce621c387f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0f985502107744f48cf56ffd8e5efa63] 2024-11-10T19:18:51,616 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting b9fab8cde9bd474cb50a2f9a8fb87b08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731266327101 2024-11-10T19:18:51,616 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 998e5e5ba4d14569af2639b03b11f7ad, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731266327101 2024-11-10T19:18:51,616 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d473bacada84d09b86a11ce621c387f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731266327737 2024-11-10T19:18:51,616 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f985502107744f48cf56ffd8e5efa63, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731266330016 2024-11-10T19:18:51,617 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 06234cdd19504f97b114da8abc86687f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731266327737 2024-11-10T19:18:51,617 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dbcd9b9396b4653bf4f0fc879d15b93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731266330016 2024-11-10T19:18:51,621 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:51,625 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#B#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:51,626 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/ccd91b97f3814238b2a0cd8dde79a5ad is 50, key is test_row_0/B:col10/1731266330017/Put/seqid=0 2024-11-10T19:18:51,642 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411103c0ea7a8481e49189369f2929ba05114_4988256ba260015146113518269b8182 store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:51,644 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411103c0ea7a8481e49189369f2929ba05114_4988256ba260015146113518269b8182, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:51,644 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103c0ea7a8481e49189369f2929ba05114_4988256ba260015146113518269b8182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:51,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742415_1591 (size=4469) 2024-11-10T19:18:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:51,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742414_1590 (size=12391) 2024-11-10T19:18:51,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T19:18:51,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:51,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:51,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:51,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:51,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:51,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:51,823 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/ccd91b97f3814238b2a0cd8dde79a5ad as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ccd91b97f3814238b2a0cd8dde79a5ad 2024-11-10T19:18:51,832 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4988256ba260015146113518269b8182/B of 4988256ba260015146113518269b8182 into ccd91b97f3814238b2a0cd8dde79a5ad(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:51,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:51,832 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/B, priority=13, startTime=1731266331614; duration=0sec 2024-11-10T19:18:51,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:51,832 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:B 2024-11-10T19:18:51,833 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:51,833 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:51,834 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/C is initiating minor compaction (all files) 2024-11-10T19:18:51,834 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/C in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:51,834 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/06aea2bf80d34802846640a517fa8ed5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/3082f6699dbe498bbafbd25384e41b5c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/92f1717d90544e9ab39e84950496861f] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=35.4 K 2024-11-10T19:18:51,834 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 06aea2bf80d34802846640a517fa8ed5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731266327101 2024-11-10T19:18:51,835 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3082f6699dbe498bbafbd25384e41b5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731266327737 2024-11-10T19:18:51,835 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f1717d90544e9ab39e84950496861f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731266330016 2024-11-10T19:18:51,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bd20113039164d40ae130d13ec5df771_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266330674/Put/seqid=0 2024-11-10T19:18:51,850 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:51,851 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/b6043dfa71c9492ba1cb1fb0af201d4a is 50, key is test_row_0/C:col10/1731266330017/Put/seqid=0 2024-11-10T19:18:51,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742416_1592 (size=12304) 2024-11-10T19:18:51,854 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:51,860 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bd20113039164d40ae130d13ec5df771_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bd20113039164d40ae130d13ec5df771_4988256ba260015146113518269b8182 2024-11-10T19:18:51,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/33344bb584f3486490b83838e4f8837e, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:51,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/33344bb584f3486490b83838e4f8837e is 175, key is test_row_0/A:col10/1731266330674/Put/seqid=0 2024-11-10T19:18:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266391860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266391861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266391865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742417_1593 (size=12391) 2024-11-10T19:18:51,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742418_1594 (size=31105) 2024-11-10T19:18:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266391972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266391972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:51,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266391981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T19:18:52,131 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:52,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-10T19:18:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T19:18:52,133 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:52,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:52,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:52,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266392181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:52,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266392181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:52,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266392191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:52,217 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#A#compaction#495 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:52,218 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/b6d42d99e7a445deb96e289344295116 is 175, key is test_row_0/A:col10/1731266330017/Put/seqid=0 2024-11-10T19:18:52,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742419_1595 (size=31345) 2024-11-10T19:18:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T19:18:52,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T19:18:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,294 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/b6043dfa71c9492ba1cb1fb0af201d4a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/b6043dfa71c9492ba1cb1fb0af201d4a 2024-11-10T19:18:52,302 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4988256ba260015146113518269b8182/C of 4988256ba260015146113518269b8182 into b6043dfa71c9492ba1cb1fb0af201d4a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:52,302 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:52,302 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/C, priority=13, startTime=1731266331615; duration=0sec 2024-11-10T19:18:52,302 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:52,302 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:C 2024-11-10T19:18:52,316 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/33344bb584f3486490b83838e4f8837e 2024-11-10T19:18:52,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/4a6e482b5b714f9b9b7f24e9e6eff7ee is 50, key is test_row_0/B:col10/1731266330674/Put/seqid=0 2024-11-10T19:18:52,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742420_1596 (size=12151) 2024-11-10T19:18:52,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T19:18:52,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T19:18:52,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266392491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:52,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266392491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266392501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:52,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T19:18:52,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:52,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,625 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/b6d42d99e7a445deb96e289344295116 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/b6d42d99e7a445deb96e289344295116 2024-11-10T19:18:52,628 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4988256ba260015146113518269b8182/A of 4988256ba260015146113518269b8182 into b6d42d99e7a445deb96e289344295116(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-10T19:18:52,628 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:52,628 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/A, priority=13, startTime=1731266331614; duration=1sec 2024-11-10T19:18:52,628 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:52,628 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:A 2024-11-10T19:18:52,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/4a6e482b5b714f9b9b7f24e9e6eff7ee 2024-11-10T19:18:52,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/6241837a5f8c4783b1b7515e0877dc2a is 50, key is test_row_0/C:col10/1731266330674/Put/seqid=0 2024-11-10T19:18:52,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T19:18:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742421_1597 (size=12151) 2024-11-10T19:18:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T19:18:52,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T19:18:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:52,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:53,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266393001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:53,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266393001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:53,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266393011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:53,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T19:18:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:53,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:53,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/6241837a5f8c4783b1b7515e0877dc2a 2024-11-10T19:18:53,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/33344bb584f3486490b83838e4f8837e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/33344bb584f3486490b83838e4f8837e 2024-11-10T19:18:53,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/33344bb584f3486490b83838e4f8837e, entries=150, sequenceid=145, filesize=30.4 K 2024-11-10T19:18:53,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/4a6e482b5b714f9b9b7f24e9e6eff7ee as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/4a6e482b5b714f9b9b7f24e9e6eff7ee 2024-11-10T19:18:53,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/4a6e482b5b714f9b9b7f24e9e6eff7ee, entries=150, sequenceid=145, filesize=11.9 K 2024-11-10T19:18:53,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/6241837a5f8c4783b1b7515e0877dc2a as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/6241837a5f8c4783b1b7515e0877dc2a 2024-11-10T19:18:53,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/6241837a5f8c4783b1b7515e0877dc2a, entries=150, sequenceid=145, filesize=11.9 K 2024-11-10T19:18:53,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4988256ba260015146113518269b8182 in 1348ms, sequenceid=145, compaction requested=false 2024-11-10T19:18:53,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:53,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T19:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:53,203 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T19:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:53,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111048bc424124cc48d5ae5c9e64a336f9f2_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266331856/Put/seqid=0 2024-11-10T19:18:53,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742422_1598 (size=12304) 2024-11-10T19:18:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:53,257 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111048bc424124cc48d5ae5c9e64a336f9f2_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111048bc424124cc48d5ae5c9e64a336f9f2_4988256ba260015146113518269b8182 2024-11-10T19:18:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/1ad8ac29fc024577b53a7bf987bb02ac, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:53,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/1ad8ac29fc024577b53a7bf987bb02ac is 175, key is test_row_0/A:col10/1731266331856/Put/seqid=0 2024-11-10T19:18:53,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T19:18:53,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742423_1599 (size=31105) 2024-11-10T19:18:53,293 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/1ad8ac29fc024577b53a7bf987bb02ac 2024-11-10T19:18:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/99ecbce75f054f639076becb53a584a4 is 50, key is test_row_0/B:col10/1731266331856/Put/seqid=0 2024-11-10T19:18:53,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742424_1600 (size=12151) 2024-11-10T19:18:53,710 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/99ecbce75f054f639076becb53a584a4 2024-11-10T19:18:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/78db2149187a4ecb88c540907f6634ef is 50, key is test_row_0/C:col10/1731266331856/Put/seqid=0 2024-11-10T19:18:53,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742425_1601 (size=12151) 2024-11-10T19:18:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:54,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:54,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266394031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266394033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266394034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,119 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/78db2149187a4ecb88c540907f6634ef 2024-11-10T19:18:54,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/1ad8ac29fc024577b53a7bf987bb02ac as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1ad8ac29fc024577b53a7bf987bb02ac 2024-11-10T19:18:54,136 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1ad8ac29fc024577b53a7bf987bb02ac, entries=150, sequenceid=168, filesize=30.4 K 2024-11-10T19:18:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/99ecbce75f054f639076becb53a584a4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/99ecbce75f054f639076becb53a584a4 2024-11-10T19:18:54,140 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/99ecbce75f054f639076becb53a584a4, entries=150, sequenceid=168, filesize=11.9 K 2024-11-10T19:18:54,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/78db2149187a4ecb88c540907f6634ef as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/78db2149187a4ecb88c540907f6634ef 2024-11-10T19:18:54,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266394142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266394142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266394142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,145 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/78db2149187a4ecb88c540907f6634ef, entries=150, sequenceid=168, filesize=11.9 K 2024-11-10T19:18:54,146 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4988256ba260015146113518269b8182 in 943ms, sequenceid=168, compaction requested=true 2024-11-10T19:18:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-10T19:18:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-10T19:18:54,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-10T19:18:54,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0130 sec 2024-11-10T19:18:54,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.0180 sec 2024-11-10T19:18:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T19:18:54,281 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:54,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-10T19:18:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T19:18:54,283 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:54,284 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:54,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:54,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T19:18:54,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:54,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:54,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:54,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:54,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:54,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:54,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110407be605e49a4d2b8692dadf76f9b5ce_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266334353/Put/seqid=0 2024-11-10T19:18:54,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742426_1602 (size=17284) 2024-11-10T19:18:54,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266394381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T19:18:54,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266394386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266394388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:54,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:54,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:54,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:54,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266394491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266394501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266394502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:54,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T19:18:54,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266394701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266394711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266394711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:54,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:54,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:54,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:54,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,770 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:54,772 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110407be605e49a4d2b8692dadf76f9b5ce_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110407be605e49a4d2b8692dadf76f9b5ce_4988256ba260015146113518269b8182 2024-11-10T19:18:54,773 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/f0d49873c3a1433dbb86ddda5638106b, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:54,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/f0d49873c3a1433dbb86ddda5638106b is 175, key is test_row_0/A:col10/1731266334353/Put/seqid=0 2024-11-10T19:18:54,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742427_1603 (size=48389) 2024-11-10T19:18:54,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:54,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T19:18:55,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266395011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:55,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266395038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:55,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266395038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:55,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,179 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/f0d49873c3a1433dbb86ddda5638106b 2024-11-10T19:18:55,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/a883575f9d8c4df29c7da8ee3775c283 is 50, key is test_row_0/B:col10/1731266334353/Put/seqid=0 2024-11-10T19:18:55,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742428_1604 (size=12151) 2024-11-10T19:18:55,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1731266395321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:55,321 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_2' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1731266395331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:55,332 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:55,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:55,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:55,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T19:18:55,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266395521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:55,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266395552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:55,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266395552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:55,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/a883575f9d8c4df29c7da8ee3775c283 2024-11-10T19:18:55,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/19e5bbdc8e2c4467807764eaa2aa68e4 is 50, key is test_row_0/C:col10/1731266334353/Put/seqid=0 2024-11-10T19:18:55,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742429_1605 (size=12151) 2024-11-10T19:18:55,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:55,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:55,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:55,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:56,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/19e5bbdc8e2c4467807764eaa2aa68e4 2024-11-10T19:18:56,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/f0d49873c3a1433dbb86ddda5638106b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/f0d49873c3a1433dbb86ddda5638106b 2024-11-10T19:18:56,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/f0d49873c3a1433dbb86ddda5638106b, entries=250, sequenceid=182, filesize=47.3 K 2024-11-10T19:18:56,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/a883575f9d8c4df29c7da8ee3775c283 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a883575f9d8c4df29c7da8ee3775c283 2024-11-10T19:18:56,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a883575f9d8c4df29c7da8ee3775c283, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T19:18:56,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/19e5bbdc8e2c4467807764eaa2aa68e4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/19e5bbdc8e2c4467807764eaa2aa68e4 2024-11-10T19:18:56,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/19e5bbdc8e2c4467807764eaa2aa68e4, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T19:18:56,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4988256ba260015146113518269b8182 in 1666ms, sequenceid=182, compaction requested=true 2024-11-10T19:18:56,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:56,021 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:56,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:56,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:56,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:56,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:56,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:56,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T19:18:56,021 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:56,022 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141944 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:56,022 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/A is initiating minor compaction (all files) 2024-11-10T19:18:56,022 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/A in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:56,022 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/b6d42d99e7a445deb96e289344295116, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/33344bb584f3486490b83838e4f8837e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1ad8ac29fc024577b53a7bf987bb02ac, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/f0d49873c3a1433dbb86ddda5638106b] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=138.6 K 2024-11-10T19:18:56,022 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:56,022 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/b6d42d99e7a445deb96e289344295116, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/33344bb584f3486490b83838e4f8837e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1ad8ac29fc024577b53a7bf987bb02ac, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/f0d49873c3a1433dbb86ddda5638106b] 2024-11-10T19:18:56,023 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6d42d99e7a445deb96e289344295116, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731266330016 2024-11-10T19:18:56,024 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:56,024 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/B is initiating minor compaction (all files) 2024-11-10T19:18:56,024 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/B in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:56,024 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ccd91b97f3814238b2a0cd8dde79a5ad, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/4a6e482b5b714f9b9b7f24e9e6eff7ee, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/99ecbce75f054f639076becb53a584a4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a883575f9d8c4df29c7da8ee3775c283] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=47.7 K 2024-11-10T19:18:56,025 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33344bb584f3486490b83838e4f8837e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731266330674 2024-11-10T19:18:56,025 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd91b97f3814238b2a0cd8dde79a5ad, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731266330016 2024-11-10T19:18:56,025 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ad8ac29fc024577b53a7bf987bb02ac, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731266331850 2024-11-10T19:18:56,026 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a6e482b5b714f9b9b7f24e9e6eff7ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731266330674 2024-11-10T19:18:56,026 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0d49873c3a1433dbb86ddda5638106b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731266334026 2024-11-10T19:18:56,027 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ecbce75f054f639076becb53a584a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731266331850 2024-11-10T19:18:56,027 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting a883575f9d8c4df29c7da8ee3775c283, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731266334030 2024-11-10T19:18:56,049 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:56,057 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411109ac785c86bbc4f0283e3cc7d6bee151d_4988256ba260015146113518269b8182 store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:56,059 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#B#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:56,059 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/b09a652740db404a9638816a3883ff90 is 50, key is test_row_0/B:col10/1731266334353/Put/seqid=0 2024-11-10T19:18:56,059 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411109ac785c86bbc4f0283e3cc7d6bee151d_4988256ba260015146113518269b8182, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:56,060 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109ac785c86bbc4f0283e3cc7d6bee151d_4988256ba260015146113518269b8182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:56,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742430_1606 (size=12527) 2024-11-10T19:18:56,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742431_1607 (size=4469) 2024-11-10T19:18:56,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T19:18:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:56,129 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T19:18:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e6a9c85c9c14f5eb89f21e4b8286f3d_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266334386/Put/seqid=0 2024-11-10T19:18:56,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742432_1608 (size=12304) 2024-11-10T19:18:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T19:18:56,485 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:56,485 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0cd251d916c54c30bc45b2b68b7414fd is 175, key is test_row_0/A:col10/1731266334353/Put/seqid=0 2024-11-10T19:18:56,489 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/b09a652740db404a9638816a3883ff90 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b09a652740db404a9638816a3883ff90 2024-11-10T19:18:56,493 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/B of 4988256ba260015146113518269b8182 into b09a652740db404a9638816a3883ff90(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:56,493 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:56,493 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/B, priority=12, startTime=1731266336021; duration=0sec 2024-11-10T19:18:56,493 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:56,493 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:B 2024-11-10T19:18:56,494 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:18:56,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742433_1609 (size=31481) 2024-11-10T19:18:56,494 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:18:56,495 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/C is initiating minor compaction (all files) 2024-11-10T19:18:56,495 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/C in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:56,495 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/b6043dfa71c9492ba1cb1fb0af201d4a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/6241837a5f8c4783b1b7515e0877dc2a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/78db2149187a4ecb88c540907f6634ef, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/19e5bbdc8e2c4467807764eaa2aa68e4] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=47.7 K 2024-11-10T19:18:56,496 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting b6043dfa71c9492ba1cb1fb0af201d4a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731266330016 2024-11-10T19:18:56,497 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6241837a5f8c4783b1b7515e0877dc2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731266330674 2024-11-10T19:18:56,497 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 78db2149187a4ecb88c540907f6634ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731266331850 2024-11-10T19:18:56,497 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e5bbdc8e2c4467807764eaa2aa68e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731266334030 2024-11-10T19:18:56,509 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:56,509 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/760e4270d036481ca9da0b4714bcff82 is 50, key is test_row_0/C:col10/1731266334353/Put/seqid=0 2024-11-10T19:18:56,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742434_1610 (size=12527) 2024-11-10T19:18:56,518 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/760e4270d036481ca9da0b4714bcff82 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/760e4270d036481ca9da0b4714bcff82 2024-11-10T19:18:56,525 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/C of 4988256ba260015146113518269b8182 into 760e4270d036481ca9da0b4714bcff82(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:56,526 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:56,526 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/C, priority=12, startTime=1731266336021; duration=0sec 2024-11-10T19:18:56,526 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:56,526 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:C 2024-11-10T19:18:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:56,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:56,557 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e6a9c85c9c14f5eb89f21e4b8286f3d_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e6a9c85c9c14f5eb89f21e4b8286f3d_4988256ba260015146113518269b8182 2024-11-10T19:18:56,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/54bfbf17732147f1a15f850e0ce1f338, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:56,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/54bfbf17732147f1a15f850e0ce1f338 is 175, key is test_row_0/A:col10/1731266334386/Put/seqid=0 2024-11-10T19:18:56,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742435_1611 (size=31105) 2024-11-10T19:18:56,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266396571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:56,575 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:56,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266396572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:56,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266396573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:56,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266396681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:56,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266396681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:56,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266396891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:56,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266396891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:56,897 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0cd251d916c54c30bc45b2b68b7414fd as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0cd251d916c54c30bc45b2b68b7414fd 2024-11-10T19:18:56,901 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/A of 4988256ba260015146113518269b8182 into 0cd251d916c54c30bc45b2b68b7414fd(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:56,902 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:56,902 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/A, priority=12, startTime=1731266336020; duration=0sec 2024-11-10T19:18:56,902 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:56,902 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:A 2024-11-10T19:18:56,962 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/54bfbf17732147f1a15f850e0ce1f338 2024-11-10T19:18:56,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/3bc7df88e80849af8ebc83da8fb80967 is 50, key is test_row_0/B:col10/1731266334386/Put/seqid=0 2024-11-10T19:18:56,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742436_1612 (size=12151) 2024-11-10T19:18:57,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266397201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266397201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:57,390 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/3bc7df88e80849af8ebc83da8fb80967 2024-11-10T19:18:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/c03c7c9ec65c4761a02726bdc1c9ae76 is 50, key is test_row_0/C:col10/1731266334386/Put/seqid=0 2024-11-10T19:18:57,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742437_1613 (size=12151) 2024-11-10T19:18:57,399 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/c03c7c9ec65c4761a02726bdc1c9ae76 2024-11-10T19:18:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/54bfbf17732147f1a15f850e0ce1f338 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/54bfbf17732147f1a15f850e0ce1f338 2024-11-10T19:18:57,406 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/54bfbf17732147f1a15f850e0ce1f338, entries=150, sequenceid=200, filesize=30.4 K 2024-11-10T19:18:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/3bc7df88e80849af8ebc83da8fb80967 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3bc7df88e80849af8ebc83da8fb80967 2024-11-10T19:18:57,409 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3bc7df88e80849af8ebc83da8fb80967, entries=150, sequenceid=200, filesize=11.9 K 2024-11-10T19:18:57,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/c03c7c9ec65c4761a02726bdc1c9ae76 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/c03c7c9ec65c4761a02726bdc1c9ae76 2024-11-10T19:18:57,412 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/c03c7c9ec65c4761a02726bdc1c9ae76, entries=150, sequenceid=200, filesize=11.9 K 2024-11-10T19:18:57,413 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4988256ba260015146113518269b8182 in 1283ms, sequenceid=200, compaction requested=false 2024-11-10T19:18:57,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:57,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:57,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-10T19:18:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-10T19:18:57,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-10T19:18:57,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1300 sec 2024-11-10T19:18:57,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.1350 sec 2024-11-10T19:18:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:57,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T19:18:57,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:57,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:57,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:57,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:57,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:57,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:57,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101b00b82293864f489d88bae82a737575_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266337714/Put/seqid=0 2024-11-10T19:18:57,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742438_1614 (size=14794) 2024-11-10T19:18:57,727 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:57,732 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101b00b82293864f489d88bae82a737575_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101b00b82293864f489d88bae82a737575_4988256ba260015146113518269b8182 2024-11-10T19:18:57,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7d756fb108a640deb1b60408e96a1dd4, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:57,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7d756fb108a640deb1b60408e96a1dd4 is 175, key is test_row_0/A:col10/1731266337714/Put/seqid=0 2024-11-10T19:18:57,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742439_1615 (size=39749) 2024-11-10T19:18:57,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266397759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:57,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266397762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:57,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266397871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:57,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266397873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266398081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:58,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266398082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7d756fb108a640deb1b60408e96a1dd4 2024-11-10T19:18:58,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/8e3d963b6b7e4c2e8eb3a239f115a61c is 50, key is test_row_0/B:col10/1731266337714/Put/seqid=0 2024-11-10T19:18:58,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742440_1616 (size=12151) 2024-11-10T19:18:58,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/8e3d963b6b7e4c2e8eb3a239f115a61c 2024-11-10T19:18:58,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/21261ed363e845d8a5c1ea16878dc358 is 50, key is test_row_0/C:col10/1731266337714/Put/seqid=0 2024-11-10T19:18:58,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742441_1617 (size=12151) 2024-11-10T19:18:58,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/21261ed363e845d8a5c1ea16878dc358 2024-11-10T19:18:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/7d756fb108a640deb1b60408e96a1dd4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d756fb108a640deb1b60408e96a1dd4 2024-11-10T19:18:58,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d756fb108a640deb1b60408e96a1dd4, entries=200, sequenceid=219, filesize=38.8 K 2024-11-10T19:18:58,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/8e3d963b6b7e4c2e8eb3a239f115a61c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8e3d963b6b7e4c2e8eb3a239f115a61c 2024-11-10T19:18:58,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8e3d963b6b7e4c2e8eb3a239f115a61c, entries=150, sequenceid=219, filesize=11.9 K 2024-11-10T19:18:58,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/21261ed363e845d8a5c1ea16878dc358 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/21261ed363e845d8a5c1ea16878dc358 2024-11-10T19:18:58,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/21261ed363e845d8a5c1ea16878dc358, entries=150, sequenceid=219, filesize=11.9 K 2024-11-10T19:18:58,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4988256ba260015146113518269b8182 in 478ms, sequenceid=219, compaction requested=true 2024-11-10T19:18:58,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:18:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:58,193 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:18:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:58,193 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:18:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:58,194 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:58,194 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:58,194 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/A is initiating minor compaction (all files) 2024-11-10T19:18:58,194 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/B is initiating minor compaction (all files) 2024-11-10T19:18:58,194 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/B in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,194 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/A in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,194 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b09a652740db404a9638816a3883ff90, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3bc7df88e80849af8ebc83da8fb80967, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8e3d963b6b7e4c2e8eb3a239f115a61c] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=36.0 K 2024-11-10T19:18:58,194 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0cd251d916c54c30bc45b2b68b7414fd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/54bfbf17732147f1a15f850e0ce1f338, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d756fb108a640deb1b60408e96a1dd4] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=99.9 K 2024-11-10T19:18:58,194 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,194 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0cd251d916c54c30bc45b2b68b7414fd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/54bfbf17732147f1a15f850e0ce1f338, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d756fb108a640deb1b60408e96a1dd4] 2024-11-10T19:18:58,194 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting b09a652740db404a9638816a3883ff90, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731266334030 2024-11-10T19:18:58,194 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cd251d916c54c30bc45b2b68b7414fd, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731266334030 2024-11-10T19:18:58,194 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc7df88e80849af8ebc83da8fb80967, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731266334380 2024-11-10T19:18:58,194 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54bfbf17732147f1a15f850e0ce1f338, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731266334380 2024-11-10T19:18:58,195 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3d963b6b7e4c2e8eb3a239f115a61c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731266336565 2024-11-10T19:18:58,195 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d756fb108a640deb1b60408e96a1dd4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731266336565 2024-11-10T19:18:58,208 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#B#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:58,208 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/842656c8cdfa4488aaa0691a4f1571fa is 50, key is test_row_0/B:col10/1731266337714/Put/seqid=0 2024-11-10T19:18:58,213 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:58,231 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110aba9a21c101e4ae695730d5b89e691d0_4988256ba260015146113518269b8182 store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:58,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742442_1618 (size=12629) 2024-11-10T19:18:58,233 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110aba9a21c101e4ae695730d5b89e691d0_4988256ba260015146113518269b8182, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:58,234 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110aba9a21c101e4ae695730d5b89e691d0_4988256ba260015146113518269b8182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:58,242 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/842656c8cdfa4488aaa0691a4f1571fa as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/842656c8cdfa4488aaa0691a4f1571fa 2024-11-10T19:18:58,247 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4988256ba260015146113518269b8182/B of 4988256ba260015146113518269b8182 into 842656c8cdfa4488aaa0691a4f1571fa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:58,247 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:58,247 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/B, priority=13, startTime=1731266338193; duration=0sec 2024-11-10T19:18:58,247 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:18:58,247 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:B 2024-11-10T19:18:58,247 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T19:18:58,248 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T19:18:58,248 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/C is initiating minor compaction (all files) 2024-11-10T19:18:58,248 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/C in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,248 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/760e4270d036481ca9da0b4714bcff82, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/c03c7c9ec65c4761a02726bdc1c9ae76, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/21261ed363e845d8a5c1ea16878dc358] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=36.0 K 2024-11-10T19:18:58,249 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 760e4270d036481ca9da0b4714bcff82, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731266334030 2024-11-10T19:18:58,249 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting c03c7c9ec65c4761a02726bdc1c9ae76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731266334380 2024-11-10T19:18:58,249 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 21261ed363e845d8a5c1ea16878dc358, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731266336565 2024-11-10T19:18:58,254 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#C#compaction#518 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:58,255 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/179ca4dc481443ba8e5f20b823f68b08 is 50, key is test_row_0/C:col10/1731266337714/Put/seqid=0 2024-11-10T19:18:58,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742443_1619 (size=4469) 2024-11-10T19:18:58,257 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#A#compaction#517 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:18:58,257 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/1aca7f0f4f484642bc7fa577fb25d40d is 175, key is test_row_0/A:col10/1731266337714/Put/seqid=0 2024-11-10T19:18:58,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742444_1620 (size=12629) 2024-11-10T19:18:58,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742445_1621 (size=31583) 2024-11-10T19:18:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:58,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:18:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:58,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100c8fc3fdd85449e9af0b04749d98e538_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266338393/Put/seqid=0 2024-11-10T19:18:58,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742446_1622 (size=17284) 2024-11-10T19:18:58,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:58,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266398421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100c8fc3fdd85449e9af0b04749d98e538_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100c8fc3fdd85449e9af0b04749d98e538_4988256ba260015146113518269b8182 2024-11-10T19:18:58,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/c85fabb85cc14833bd116e80cc95720c, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:58,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/c85fabb85cc14833bd116e80cc95720c is 175, key is test_row_0/A:col10/1731266338393/Put/seqid=0 2024-11-10T19:18:58,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266398427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742447_1623 (size=48389) 2024-11-10T19:18:58,441 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/c85fabb85cc14833bd116e80cc95720c 2024-11-10T19:18:58,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/ce57059627a74fcb801e245945f8dc8e is 50, key is test_row_0/B:col10/1731266338393/Put/seqid=0 2024-11-10T19:18:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T19:18:58,451 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:18:58,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:18:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-10T19:18:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T19:18:58,468 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:18:58,469 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:18:58,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:18:58,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742448_1624 (size=12151) 2024-11-10T19:18:58,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266398531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:58,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266398542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T19:18:58,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266398581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:58,585 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:18:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T19:18:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:58,674 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/179ca4dc481443ba8e5f20b823f68b08 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/179ca4dc481443ba8e5f20b823f68b08 2024-11-10T19:18:58,679 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4988256ba260015146113518269b8182/C of 4988256ba260015146113518269b8182 into 179ca4dc481443ba8e5f20b823f68b08(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:58,679 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:58,679 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/C, priority=13, startTime=1731266338193; duration=0sec 2024-11-10T19:18:58,679 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:58,679 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:C 2024-11-10T19:18:58,699 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/1aca7f0f4f484642bc7fa577fb25d40d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1aca7f0f4f484642bc7fa577fb25d40d 2024-11-10T19:18:58,702 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4988256ba260015146113518269b8182/A of 4988256ba260015146113518269b8182 into 1aca7f0f4f484642bc7fa577fb25d40d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:18:58,703 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:58,703 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/A, priority=13, startTime=1731266338193; duration=0sec 2024-11-10T19:18:58,703 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:18:58,703 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:A 2024-11-10T19:18:58,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266398741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266398751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:58,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:58,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:58,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T19:18:58,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:58,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:58,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T19:18:58,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/ce57059627a74fcb801e245945f8dc8e 2024-11-10T19:18:58,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/4a297515620a427e837f4316cc4e47c2 is 50, key is test_row_0/C:col10/1731266338393/Put/seqid=0 2024-11-10T19:18:58,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742449_1625 (size=12151) 2024-11-10T19:18:58,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T19:18:58,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:58,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:58,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:58,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:59,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266399052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:59,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266399064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:59,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:59,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T19:18:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:59,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T19:18:59,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T19:18:59,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:59,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:59,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:59,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:59,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:18:59,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/4a297515620a427e837f4316cc4e47c2 2024-11-10T19:18:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/c85fabb85cc14833bd116e80cc95720c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/c85fabb85cc14833bd116e80cc95720c 2024-11-10T19:18:59,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/c85fabb85cc14833bd116e80cc95720c, entries=250, sequenceid=238, filesize=47.3 K 2024-11-10T19:18:59,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/ce57059627a74fcb801e245945f8dc8e as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ce57059627a74fcb801e245945f8dc8e 2024-11-10T19:18:59,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ce57059627a74fcb801e245945f8dc8e, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T19:18:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/4a297515620a427e837f4316cc4e47c2 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/4a297515620a427e837f4316cc4e47c2 2024-11-10T19:18:59,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/4a297515620a427e837f4316cc4e47c2, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T19:18:59,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4988256ba260015146113518269b8182 in 936ms, sequenceid=238, compaction requested=false 2024-11-10T19:18:59,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:59,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T19:18:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:59,385 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T19:18:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:59,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:59,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dcb6e43aae3847979ab41ae746240aa4_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266338420/Put/seqid=0 2024-11-10T19:18:59,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742450_1626 (size=12304) 2024-11-10T19:18:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:59,397 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dcb6e43aae3847979ab41ae746240aa4_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dcb6e43aae3847979ab41ae746240aa4_4988256ba260015146113518269b8182 2024-11-10T19:18:59,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/737f22cc5eb940f18ad02fd71cf23ac1, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:59,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/737f22cc5eb940f18ad02fd71cf23ac1 is 175, key is test_row_0/A:col10/1731266338420/Put/seqid=0 2024-11-10T19:18:59,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742451_1627 (size=31105) 2024-11-10T19:18:59,401 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/737f22cc5eb940f18ad02fd71cf23ac1 2024-11-10T19:18:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/00286edd7fc247919706161a47b3a461 is 50, key is test_row_0/B:col10/1731266338420/Put/seqid=0 2024-11-10T19:18:59,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742452_1628 (size=12151) 2024-11-10T19:18:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:18:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T19:18:59,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266399608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:59,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266399609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:59,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:59,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266399721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266399721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:59,810 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/00286edd7fc247919706161a47b3a461 2024-11-10T19:18:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/1503335f8a484aeea907cee6335bab38 is 50, key is test_row_0/C:col10/1731266338420/Put/seqid=0 2024-11-10T19:18:59,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742453_1629 (size=12151) 2024-11-10T19:18:59,826 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/1503335f8a484aeea907cee6335bab38 2024-11-10T19:18:59,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/737f22cc5eb940f18ad02fd71cf23ac1 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/737f22cc5eb940f18ad02fd71cf23ac1 2024-11-10T19:18:59,833 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/737f22cc5eb940f18ad02fd71cf23ac1, entries=150, sequenceid=254, filesize=30.4 K 2024-11-10T19:18:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/00286edd7fc247919706161a47b3a461 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/00286edd7fc247919706161a47b3a461 2024-11-10T19:18:59,836 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/00286edd7fc247919706161a47b3a461, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T19:18:59,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/1503335f8a484aeea907cee6335bab38 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/1503335f8a484aeea907cee6335bab38 2024-11-10T19:18:59,840 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/1503335f8a484aeea907cee6335bab38, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T19:18:59,841 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4988256ba260015146113518269b8182 in 456ms, sequenceid=254, compaction requested=true 2024-11-10T19:18:59,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:18:59,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:18:59,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-10T19:18:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-10T19:18:59,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-10T19:18:59,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3730 sec 2024-11-10T19:18:59,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.3930 sec 2024-11-10T19:18:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:18:59,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T19:18:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:18:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:18:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:18:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:18:59,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110395b1e6034a9444cb66ef17a37e814c2_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266339593/Put/seqid=0 2024-11-10T19:18:59,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742454_1630 (size=14994) 2024-11-10T19:18:59,954 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:18:59,958 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110395b1e6034a9444cb66ef17a37e814c2_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110395b1e6034a9444cb66ef17a37e814c2_4988256ba260015146113518269b8182 2024-11-10T19:18:59,959 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/3e185a940e844663a1d4a837407b9992, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:18:59,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/3e185a940e844663a1d4a837407b9992 is 175, key is test_row_0/A:col10/1731266339593/Put/seqid=0 2024-11-10T19:18:59,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266399959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:18:59,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742455_1631 (size=39949) 2024-11-10T19:18:59,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:18:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266399962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:18:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:18:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266400071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266400071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266400284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266400284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,366 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/3e185a940e844663a1d4a837407b9992 2024-11-10T19:19:00,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/3621004ee4f3425d9128557a3ba3bd33 is 50, key is test_row_0/B:col10/1731266339593/Put/seqid=0 2024-11-10T19:19:00,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742456_1632 (size=12301) 2024-11-10T19:19:00,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/3621004ee4f3425d9128557a3ba3bd33 2024-11-10T19:19:00,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/24783a3506354b3288261cd4de02a4e0 is 50, key is test_row_0/C:col10/1731266339593/Put/seqid=0 2024-11-10T19:19:00,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742457_1633 (size=12301) 2024-11-10T19:19:00,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/24783a3506354b3288261cd4de02a4e0 2024-11-10T19:19:00,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/3e185a940e844663a1d4a837407b9992 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/3e185a940e844663a1d4a837407b9992 2024-11-10T19:19:00,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/3e185a940e844663a1d4a837407b9992, entries=200, sequenceid=273, filesize=39.0 K 2024-11-10T19:19:00,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/3621004ee4f3425d9128557a3ba3bd33 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3621004ee4f3425d9128557a3ba3bd33 2024-11-10T19:19:00,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3621004ee4f3425d9128557a3ba3bd33, entries=150, sequenceid=273, filesize=12.0 K 2024-11-10T19:19:00,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/24783a3506354b3288261cd4de02a4e0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/24783a3506354b3288261cd4de02a4e0 2024-11-10T19:19:00,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/24783a3506354b3288261cd4de02a4e0, entries=150, sequenceid=273, filesize=12.0 K 2024-11-10T19:19:00,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4988256ba260015146113518269b8182 in 524ms, sequenceid=273, compaction requested=true 2024-11-10T19:19:00,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:00,457 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:19:00,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:19:00,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:19:00,457 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:19:00,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:19:00,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:19:00,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:19:00,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:19:00,458 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151026 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:19:00,459 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/A is initiating minor compaction (all files) 2024-11-10T19:19:00,459 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/A in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:00,459 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1aca7f0f4f484642bc7fa577fb25d40d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/c85fabb85cc14833bd116e80cc95720c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/737f22cc5eb940f18ad02fd71cf23ac1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/3e185a940e844663a1d4a837407b9992] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=147.5 K 2024-11-10T19:19:00,459 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:00,459 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1aca7f0f4f484642bc7fa577fb25d40d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/c85fabb85cc14833bd116e80cc95720c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/737f22cc5eb940f18ad02fd71cf23ac1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/3e185a940e844663a1d4a837407b9992] 2024-11-10T19:19:00,459 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:19:00,459 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/B is initiating minor compaction (all files) 2024-11-10T19:19:00,459 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/B in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:00,459 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/842656c8cdfa4488aaa0691a4f1571fa, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ce57059627a74fcb801e245945f8dc8e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/00286edd7fc247919706161a47b3a461, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3621004ee4f3425d9128557a3ba3bd33] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=48.1 K 2024-11-10T19:19:00,460 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aca7f0f4f484642bc7fa577fb25d40d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731266336565 2024-11-10T19:19:00,460 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 842656c8cdfa4488aaa0691a4f1571fa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731266336565 2024-11-10T19:19:00,460 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting c85fabb85cc14833bd116e80cc95720c, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731266337730 2024-11-10T19:19:00,461 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting ce57059627a74fcb801e245945f8dc8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731266337761 2024-11-10T19:19:00,461 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737f22cc5eb940f18ad02fd71cf23ac1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731266338419 2024-11-10T19:19:00,461 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 00286edd7fc247919706161a47b3a461, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731266338419 2024-11-10T19:19:00,461 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e185a940e844663a1d4a837407b9992, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731266339593 2024-11-10T19:19:00,462 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 3621004ee4f3425d9128557a3ba3bd33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731266339593 2024-11-10T19:19:00,481 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:00,486 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#B#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:19:00,487 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/61cd3f24584a4776acfc460f4a62eae7 is 50, key is test_row_0/B:col10/1731266339593/Put/seqid=0 2024-11-10T19:19:00,525 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111005a61a70d8a94d0c95996fa201352707_4988256ba260015146113518269b8182 store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:00,527 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111005a61a70d8a94d0c95996fa201352707_4988256ba260015146113518269b8182, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:00,528 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111005a61a70d8a94d0c95996fa201352707_4988256ba260015146113518269b8182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:00,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742458_1634 (size=12915) 2024-11-10T19:19:00,545 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/61cd3f24584a4776acfc460f4a62eae7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/61cd3f24584a4776acfc460f4a62eae7 2024-11-10T19:19:00,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742459_1635 (size=4469) 2024-11-10T19:19:00,552 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/B of 4988256ba260015146113518269b8182 into 61cd3f24584a4776acfc460f4a62eae7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:19:00,552 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:00,552 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/B, priority=12, startTime=1731266340457; duration=0sec 2024-11-10T19:19:00,552 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:19:00,552 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:B 2024-11-10T19:19:00,552 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:19:00,553 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:19:00,553 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/C is initiating minor compaction (all files) 2024-11-10T19:19:00,554 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/C in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:00,554 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/179ca4dc481443ba8e5f20b823f68b08, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/4a297515620a427e837f4316cc4e47c2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/1503335f8a484aeea907cee6335bab38, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/24783a3506354b3288261cd4de02a4e0] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=48.1 K 2024-11-10T19:19:00,554 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 179ca4dc481443ba8e5f20b823f68b08, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731266336565 2024-11-10T19:19:00,554 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a297515620a427e837f4316cc4e47c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731266337761 2024-11-10T19:19:00,554 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 1503335f8a484aeea907cee6335bab38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731266338419 2024-11-10T19:19:00,555 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 24783a3506354b3288261cd4de02a4e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731266339593 2024-11-10T19:19:00,580 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#C#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:19:00,580 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/9352627a07434c978ec44cd091f0ab1b is 50, key is test_row_0/C:col10/1731266339593/Put/seqid=0 2024-11-10T19:19:00,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742460_1636 (size=12915) 2024-11-10T19:19:00,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T19:19:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:19:00,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:19:00,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:00,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:19:00,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:00,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:19:00,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T19:19:00,611 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:19:00,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T19:19:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-10T19:19:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T19:19:00,615 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T19:19:00,616 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T19:19:00,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T19:19:00,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111099c9e191e2f2457d89a966f687f068d8_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266340594/Put/seqid=0 2024-11-10T19:19:00,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742461_1637 (size=12454) 2024-11-10T19:19:00,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266400641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266400642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T19:19:00,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266400751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266400761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:00,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:00,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:00,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T19:19:00,951 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#A#compaction#528 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:19:00,952 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/d91dc363228146579e31608b6b2c7b28 is 175, key is test_row_0/A:col10/1731266339593/Put/seqid=0 2024-11-10T19:19:00,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742462_1638 (size=31869) 2024-11-10T19:19:00,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266400961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266400971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:00,997 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/9352627a07434c978ec44cd091f0ab1b as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/9352627a07434c978ec44cd091f0ab1b 2024-11-10T19:19:01,001 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/C of 4988256ba260015146113518269b8182 into 9352627a07434c978ec44cd091f0ab1b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:19:01,001 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:01,001 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/C, priority=12, startTime=1731266340458; duration=0sec 2024-11-10T19:19:01,001 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:19:01,001 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:C 2024-11-10T19:19:01,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:19:01,024 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111099c9e191e2f2457d89a966f687f068d8_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111099c9e191e2f2457d89a966f687f068d8_4988256ba260015146113518269b8182 2024-11-10T19:19:01,025 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/23723773ad554c778626252ca86ee73c, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:01,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/23723773ad554c778626252ca86ee73c is 175, key is test_row_0/A:col10/1731266340594/Put/seqid=0 2024-11-10T19:19:01,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742463_1639 (size=31255) 2024-11-10T19:19:01,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:01,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:01,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T19:19:01,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266401271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:01,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266401281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:01,360 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/d91dc363228146579e31608b6b2c7b28 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/d91dc363228146579e31608b6b2c7b28 2024-11-10T19:19:01,377 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/A of 4988256ba260015146113518269b8182 into d91dc363228146579e31608b6b2c7b28(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:19:01,377 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:01,378 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/A, priority=12, startTime=1731266340457; duration=0sec 2024-11-10T19:19:01,378 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:19:01,378 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:A 2024-11-10T19:19:01,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/23723773ad554c778626252ca86ee73c 2024-11-10T19:19:01,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/abfc76495434457f92d6746d567afc62 is 50, key is test_row_0/B:col10/1731266340594/Put/seqid=0 2024-11-10T19:19:01,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742464_1640 (size=12301) 2024-11-10T19:19:01,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:01,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:01,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T19:19:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266401781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:01,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266401791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:01,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:01,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/abfc76495434457f92d6746d567afc62 2024-11-10T19:19:01,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/714039d3cb514123a62c6073fab7ed05 is 50, key is test_row_0/C:col10/1731266340594/Put/seqid=0 2024-11-10T19:19:01,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742465_1641 (size=12301) 2024-11-10T19:19:01,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:01,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:01,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:01,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:02,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:02,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:02,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:02,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:02,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:02,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:19:02,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/714039d3cb514123a62c6073fab7ed05 2024-11-10T19:19:02,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/23723773ad554c778626252ca86ee73c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/23723773ad554c778626252ca86ee73c 2024-11-10T19:19:02,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/23723773ad554c778626252ca86ee73c, entries=150, sequenceid=289, filesize=30.5 K 2024-11-10T19:19:02,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/abfc76495434457f92d6746d567afc62 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/abfc76495434457f92d6746d567afc62 2024-11-10T19:19:02,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/abfc76495434457f92d6746d567afc62, entries=150, sequenceid=289, filesize=12.0 K 2024-11-10T19:19:02,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/714039d3cb514123a62c6073fab7ed05 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/714039d3cb514123a62c6073fab7ed05 2024-11-10T19:19:02,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/714039d3cb514123a62c6073fab7ed05, entries=150, sequenceid=289, filesize=12.0 K 2024-11-10T19:19:02,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4988256ba260015146113518269b8182 in 1699ms, sequenceid=289, compaction requested=false 2024-11-10T19:19:02,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:02,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34487 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T19:19:02,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:02,302 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:19:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:19:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:19:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:19:02,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:02,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110668d732c3f414e7faa42de31021b2f14_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266340635/Put/seqid=0 2024-11-10T19:19:02,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742466_1642 (size=12454) 2024-11-10T19:19:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:19:02,353 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110668d732c3f414e7faa42de31021b2f14_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110668d732c3f414e7faa42de31021b2f14_4988256ba260015146113518269b8182 2024-11-10T19:19:02,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/91315a1337954d2f89cce9140f55cfae, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:02,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/91315a1337954d2f89cce9140f55cfae is 175, key is test_row_0/A:col10/1731266340635/Put/seqid=0 2024-11-10T19:19:02,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742467_1643 (size=31255) 2024-11-10T19:19:02,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. as already flushing 2024-11-10T19:19:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:19:02,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266402663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T19:19:02,768 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/91315a1337954d2f89cce9140f55cfae 2024-11-10T19:19:02,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266402771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/65e5ca2daf4f4a38883d80cc7d22b087 is 50, key is test_row_0/B:col10/1731266340635/Put/seqid=0 2024-11-10T19:19:02,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742468_1644 (size=12301) 2024-11-10T19:19:02,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48118 deadline: 1731266402791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:02,794 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_1' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:19:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:02,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1731266402811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:02,815 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bdc25462c9ef,34487,1731266167729 for 'test_row_0' in 4988256ba260015146113518269b8182 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T19:19:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:02,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266402981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:03,181 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/65e5ca2daf4f4a38883d80cc7d22b087 2024-11-10T19:19:03,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/050a6716562a468e936dff0d7288ef98 is 50, key is test_row_0/C:col10/1731266340635/Put/seqid=0 2024-11-10T19:19:03,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742469_1645 (size=12301) 2024-11-10T19:19:03,247 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/050a6716562a468e936dff0d7288ef98 2024-11-10T19:19:03,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/91315a1337954d2f89cce9140f55cfae as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/91315a1337954d2f89cce9140f55cfae 2024-11-10T19:19:03,269 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/91315a1337954d2f89cce9140f55cfae, entries=150, sequenceid=308, filesize=30.5 K 2024-11-10T19:19:03,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/65e5ca2daf4f4a38883d80cc7d22b087 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/65e5ca2daf4f4a38883d80cc7d22b087 2024-11-10T19:19:03,274 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/65e5ca2daf4f4a38883d80cc7d22b087, entries=150, sequenceid=308, filesize=12.0 K 2024-11-10T19:19:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/050a6716562a468e936dff0d7288ef98 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/050a6716562a468e936dff0d7288ef98 2024-11-10T19:19:03,280 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/050a6716562a468e936dff0d7288ef98, entries=150, sequenceid=308, filesize=12.0 K 2024-11-10T19:19:03,282 INFO [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4988256ba260015146113518269b8182 in 980ms, sequenceid=308, compaction requested=true 2024-11-10T19:19:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bdc25462c9ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-10T19:19:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-10T19:19:03,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-10T19:19:03,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6680 sec 2024-11-10T19:19:03,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.6750 sec 2024-11-10T19:19:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:19:03,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T19:19:03,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:19:03,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:03,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:19:03,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:03,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:19:03,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:03,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111058c91e8df21244a0b08bd0e0c708bc67_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266343294/Put/seqid=0 2024-11-10T19:19:03,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742470_1646 (size=12454) 2024-11-10T19:19:03,343 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:19:03,346 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111058c91e8df21244a0b08bd0e0c708bc67_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111058c91e8df21244a0b08bd0e0c708bc67_4988256ba260015146113518269b8182 2024-11-10T19:19:03,347 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/df71a70812c44f5dbdb7f306cd015aaf, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:03,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/df71a70812c44f5dbdb7f306cd015aaf is 175, key is test_row_0/A:col10/1731266343294/Put/seqid=0 2024-11-10T19:19:03,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742471_1647 (size=31255) 2024-11-10T19:19:03,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266403405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:03,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266403511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:03,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:03,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:03,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:03,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266403721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:03,785 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/df71a70812c44f5dbdb7f306cd015aaf 2024-11-10T19:19:03,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/6346dd0d3f0547c78a54cd15574c756d is 50, key is test_row_0/B:col10/1731266343294/Put/seqid=0 2024-11-10T19:19:03,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742472_1648 (size=12301) 2024-11-10T19:19:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266404031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:04,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/6346dd0d3f0547c78a54cd15574c756d 2024-11-10T19:19:04,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/49f1b09f38004cdca9e29f71f6f586ab is 50, key is test_row_0/C:col10/1731266343294/Put/seqid=0 2024-11-10T19:19:04,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742473_1649 (size=12301) 2024-11-10T19:19:04,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T19:19:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48108 deadline: 1731266404541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 , the old value is region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T19:19:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., hostname=bdc25462c9ef,34487,1731266167729, seqNum=5 because the exception is null or not the one we care about 2024-11-10T19:19:04,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/49f1b09f38004cdca9e29f71f6f586ab 2024-11-10T19:19:04,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/df71a70812c44f5dbdb7f306cd015aaf as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/df71a70812c44f5dbdb7f306cd015aaf 2024-11-10T19:19:04,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/df71a70812c44f5dbdb7f306cd015aaf, entries=150, sequenceid=324, filesize=30.5 K 2024-11-10T19:19:04,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/6346dd0d3f0547c78a54cd15574c756d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/6346dd0d3f0547c78a54cd15574c756d 2024-11-10T19:19:04,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/6346dd0d3f0547c78a54cd15574c756d, entries=150, sequenceid=324, filesize=12.0 K 2024-11-10T19:19:04,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/49f1b09f38004cdca9e29f71f6f586ab as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/49f1b09f38004cdca9e29f71f6f586ab 2024-11-10T19:19:04,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/49f1b09f38004cdca9e29f71f6f586ab, entries=150, sequenceid=324, filesize=12.0 K 2024-11-10T19:19:04,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4988256ba260015146113518269b8182 in 1332ms, sequenceid=324, compaction requested=true 2024-11-10T19:19:04,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:04,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T19:19:04,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:19:04,627 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:19:04,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T19:19:04,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:19:04,627 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:19:04,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4988256ba260015146113518269b8182:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T19:19:04,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:19:04,628 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:19:04,628 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/B is initiating minor compaction (all files) 2024-11-10T19:19:04,628 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/B in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:04,628 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/61cd3f24584a4776acfc460f4a62eae7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/abfc76495434457f92d6746d567afc62, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/65e5ca2daf4f4a38883d80cc7d22b087, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/6346dd0d3f0547c78a54cd15574c756d] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=48.7 K 2024-11-10T19:19:04,629 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:19:04,629 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 61cd3f24584a4776acfc460f4a62eae7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731266339593 2024-11-10T19:19:04,629 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/A is initiating minor compaction (all files) 2024-11-10T19:19:04,629 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/A in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:04,629 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/d91dc363228146579e31608b6b2c7b28, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/23723773ad554c778626252ca86ee73c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/91315a1337954d2f89cce9140f55cfae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/df71a70812c44f5dbdb7f306cd015aaf] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=122.7 K 2024-11-10T19:19:04,629 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:04,629 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. files: [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/d91dc363228146579e31608b6b2c7b28, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/23723773ad554c778626252ca86ee73c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/91315a1337954d2f89cce9140f55cfae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/df71a70812c44f5dbdb7f306cd015aaf] 2024-11-10T19:19:04,629 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting d91dc363228146579e31608b6b2c7b28, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731266339593 2024-11-10T19:19:04,629 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting abfc76495434457f92d6746d567afc62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731266339958 2024-11-10T19:19:04,630 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23723773ad554c778626252ca86ee73c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731266339958 2024-11-10T19:19:04,630 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 65e5ca2daf4f4a38883d80cc7d22b087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731266340635 2024-11-10T19:19:04,630 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 6346dd0d3f0547c78a54cd15574c756d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731266342657 2024-11-10T19:19:04,630 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91315a1337954d2f89cce9140f55cfae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731266340635 2024-11-10T19:19:04,630 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] compactions.Compactor(225): Compacting df71a70812c44f5dbdb7f306cd015aaf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731266342657 2024-11-10T19:19:04,636 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:04,639 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#B#compaction#541 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:19:04,639 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/92f74b467c5641d69bb04637abc0c7f8 is 50, key is test_row_0/B:col10/1731266343294/Put/seqid=0 2024-11-10T19:19:04,641 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411105cd615c405a74035975944db2399b601_4988256ba260015146113518269b8182 store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:04,644 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411105cd615c405a74035975944db2399b601_4988256ba260015146113518269b8182, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:04,644 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105cd615c405a74035975944db2399b601_4988256ba260015146113518269b8182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:04,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742474_1650 (size=13051) 2024-11-10T19:19:04,670 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/92f74b467c5641d69bb04637abc0c7f8 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/92f74b467c5641d69bb04637abc0c7f8 2024-11-10T19:19:04,676 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/B of 4988256ba260015146113518269b8182 into 92f74b467c5641d69bb04637abc0c7f8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:19:04,676 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:04,676 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/B, priority=12, startTime=1731266344627; duration=0sec 2024-11-10T19:19:04,676 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T19:19:04,676 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:B 2024-11-10T19:19:04,676 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T19:19:04,677 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T19:19:04,677 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1541): 4988256ba260015146113518269b8182/C is initiating minor compaction (all files) 2024-11-10T19:19:04,677 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4988256ba260015146113518269b8182/C in TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:04,677 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/9352627a07434c978ec44cd091f0ab1b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/714039d3cb514123a62c6073fab7ed05, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/050a6716562a468e936dff0d7288ef98, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/49f1b09f38004cdca9e29f71f6f586ab] into tmpdir=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp, totalSize=48.7 K 2024-11-10T19:19:04,678 INFO [Thread-2507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2507. 2024-11-10T19:19:04,678 DEBUG [Thread-2507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:04,678 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,678 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:04,678 INFO [Thread-2505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2505. 2024-11-10T19:19:04,678 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 9352627a07434c978ec44cd091f0ab1b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731266339593 2024-11-10T19:19:04,678 DEBUG [Thread-2505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:04,678 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,678 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:04,679 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 714039d3cb514123a62c6073fab7ed05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731266339958 2024-11-10T19:19:04,679 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 050a6716562a468e936dff0d7288ef98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731266340635 2024-11-10T19:19:04,679 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f1b09f38004cdca9e29f71f6f586ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731266342657 2024-11-10T19:19:04,681 INFO [Thread-2509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2509. 2024-11-10T19:19:04,681 DEBUG [Thread-2509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:04,681 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,681 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:04,681 INFO [Thread-2501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2501. 2024-11-10T19:19:04,681 DEBUG [Thread-2501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:04,681 DEBUG [Thread-2501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,681 DEBUG [Thread-2501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:04,682 INFO [Thread-2503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2503. 2024-11-10T19:19:04,682 DEBUG [Thread-2503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:04,682 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,682 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:04,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742475_1651 (size=4469) 2024-11-10T19:19:04,683 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#A#compaction#540 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:19:04,683 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0ce7ca797d834f958d6e7d17b2d8d947 is 175, key is test_row_0/A:col10/1731266343294/Put/seqid=0 2024-11-10T19:19:04,690 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4988256ba260015146113518269b8182#C#compaction#542 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T19:19:04,690 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/977df2ec9456450eb41ee41a710695ae is 50, key is test_row_0/C:col10/1731266343294/Put/seqid=0 2024-11-10T19:19:04,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742476_1652 (size=32005) 2024-11-10T19:19:04,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742477_1653 (size=13051) 2024-11-10T19:19:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T19:19:04,772 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:19:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34487 {}] regionserver.HRegion(8855): Flush requested on 4988256ba260015146113518269b8182 2024-11-10T19:19:04,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T19:19:04,802 INFO [Thread-2492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2492. 2024-11-10T19:19:04,802 DEBUG [Thread-2492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:04,802 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:19:04,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:04,803 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:19:04,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:04,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:04,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:19:04,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:04,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110428fbf20f35040eca577c3f785f53439_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266343386/Put/seqid=0 2024-11-10T19:19:04,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742478_1654 (size=12454) 2024-11-10T19:19:04,843 INFO [Thread-2490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2490. 2024-11-10T19:19:04,843 DEBUG [Thread-2490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:04,843 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,843 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:04,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:05,095 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/0ce7ca797d834f958d6e7d17b2d8d947 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0ce7ca797d834f958d6e7d17b2d8d947 2024-11-10T19:19:05,096 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/977df2ec9456450eb41ee41a710695ae as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/977df2ec9456450eb41ee41a710695ae 2024-11-10T19:19:05,098 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/A of 4988256ba260015146113518269b8182 into 0ce7ca797d834f958d6e7d17b2d8d947(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:19:05,098 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:05,098 INFO [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/A, priority=12, startTime=1731266344627; duration=0sec 2024-11-10T19:19:05,099 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:19:05,099 DEBUG [RS:0;bdc25462c9ef:34487-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:A 2024-11-10T19:19:05,099 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4988256ba260015146113518269b8182/C of 4988256ba260015146113518269b8182 into 977df2ec9456450eb41ee41a710695ae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T19:19:05,099 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:05,099 INFO [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., storeName=4988256ba260015146113518269b8182/C, priority=12, startTime=1731266344627; duration=0sec 2024-11-10T19:19:05,099 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T19:19:05,099 DEBUG [RS:0;bdc25462c9ef:34487-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4988256ba260015146113518269b8182:C 2024-11-10T19:19:05,212 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:19:05,215 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110428fbf20f35040eca577c3f785f53439_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110428fbf20f35040eca577c3f785f53439_4988256ba260015146113518269b8182 2024-11-10T19:19:05,216 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/72b5d8f05fb543edb05dc45ce5e680f7, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:05,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/72b5d8f05fb543edb05dc45ce5e680f7 is 175, key is test_row_0/A:col10/1731266343386/Put/seqid=0 2024-11-10T19:19:05,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742479_1655 (size=31255) 2024-11-10T19:19:05,372 INFO [Thread-2496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2496. 2024-11-10T19:19:05,372 DEBUG [Thread-2496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:05,372 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:05,373 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:05,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:05,442 INFO [Thread-2494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2494. 2024-11-10T19:19:05,442 DEBUG [Thread-2494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:05,442 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:05,442 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:05,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:05,562 INFO [Thread-2498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2498. 2024-11-10T19:19:05,562 DEBUG [Thread-2498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T19:19:05,562 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:05,562 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:05,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2707 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2660 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2635 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2705 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2640 2024-11-10T19:19:05,562 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T19:19:05,563 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:19:05,563 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:19:05,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:05,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:05,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:05,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T19:19:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T19:19:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T19:19:05,566 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266345566"}]},"ts":"1731266345566"} 2024-11-10T19:19:05,567 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T19:19:05,567 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T19:19:05,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T19:19:05,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, UNASSIGN}] 2024-11-10T19:19:05,569 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, UNASSIGN 2024-11-10T19:19:05,570 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=4988256ba260015146113518269b8182, regionState=CLOSING, regionLocation=bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:05,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, UNASSIGN because future has completed 2024-11-10T19:19:05,571 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T19:19:05,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729}] 2024-11-10T19:19:05,620 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/72b5d8f05fb543edb05dc45ce5e680f7 2024-11-10T19:19:05,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/9d51d7276b4b4f20afd539ebe460b0f6 is 50, key is test_row_0/B:col10/1731266343386/Put/seqid=0 2024-11-10T19:19:05,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742480_1656 (size=12301) 2024-11-10T19:19:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T19:19:05,723 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(122): Close 4988256ba260015146113518269b8182 2024-11-10T19:19:05,723 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T19:19:05,723 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1722): Closing 4988256ba260015146113518269b8182, disabling compactions & flushes 2024-11-10T19:19:05,723 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T19:19:05,898 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T19:19:06,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/9d51d7276b4b4f20afd539ebe460b0f6 2024-11-10T19:19:06,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/d1066498e43a417686caa0d8fe86b9d7 is 50, key is test_row_0/C:col10/1731266343386/Put/seqid=0 2024-11-10T19:19:06,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742481_1657 (size=12301) 2024-11-10T19:19:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T19:19:06,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/d1066498e43a417686caa0d8fe86b9d7 2024-11-10T19:19:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/72b5d8f05fb543edb05dc45ce5e680f7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/72b5d8f05fb543edb05dc45ce5e680f7 2024-11-10T19:19:06,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/72b5d8f05fb543edb05dc45ce5e680f7, entries=150, sequenceid=342, filesize=30.5 K 2024-11-10T19:19:06,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/9d51d7276b4b4f20afd539ebe460b0f6 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/9d51d7276b4b4f20afd539ebe460b0f6 2024-11-10T19:19:06,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/9d51d7276b4b4f20afd539ebe460b0f6, entries=150, sequenceid=342, filesize=12.0 K 2024-11-10T19:19:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/d1066498e43a417686caa0d8fe86b9d7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/d1066498e43a417686caa0d8fe86b9d7 2024-11-10T19:19:06,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/d1066498e43a417686caa0d8fe86b9d7, entries=150, sequenceid=342, filesize=12.0 K 2024-11-10T19:19:06,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for 4988256ba260015146113518269b8182 in 1645ms, sequenceid=342, compaction requested=false 2024-11-10T19:19:06,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4988256ba260015146113518269b8182: 2024-11-10T19:19:06,447 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:06,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:06,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. after waiting 0 ms 2024-11-10T19:19:06,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:06,447 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(2902): Flushing 4988256ba260015146113518269b8182 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T19:19:06,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=A 2024-11-10T19:19:06,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:06,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=B 2024-11-10T19:19:06,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:06,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4988256ba260015146113518269b8182, store=C 2024-11-10T19:19:06,447 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T19:19:06,452 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d7d9fd5b338240fd89106e35bd6d7307_4988256ba260015146113518269b8182 is 50, key is test_row_0/A:col10/1731266345561/Put/seqid=0 2024-11-10T19:19:06,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742482_1658 (size=9914) 2024-11-10T19:19:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T19:19:06,855 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:19:06,857 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d7d9fd5b338240fd89106e35bd6d7307_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d7d9fd5b338240fd89106e35bd6d7307_4988256ba260015146113518269b8182 2024-11-10T19:19:06,858 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/9151545f96114815b4d9450d6066423c, store: [table=TestAcidGuarantees family=A region=4988256ba260015146113518269b8182] 2024-11-10T19:19:06,859 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/9151545f96114815b4d9450d6066423c is 175, key is test_row_0/A:col10/1731266345561/Put/seqid=0 2024-11-10T19:19:06,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742483_1659 (size=22561) 2024-11-10T19:19:07,262 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/9151545f96114815b4d9450d6066423c 2024-11-10T19:19:07,266 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/8d53f673af1e4264be49efa70a5cc321 is 50, key is test_row_0/B:col10/1731266345561/Put/seqid=0 2024-11-10T19:19:07,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742484_1660 (size=9857) 2024-11-10T19:19:07,669 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/8d53f673af1e4264be49efa70a5cc321 2024-11-10T19:19:07,674 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/83663622396a4350a7d66edbb78695f9 is 50, key is test_row_0/C:col10/1731266345561/Put/seqid=0 2024-11-10T19:19:07,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742485_1661 (size=9857) 2024-11-10T19:19:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T19:19:08,077 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/83663622396a4350a7d66edbb78695f9 2024-11-10T19:19:08,081 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/A/9151545f96114815b4d9450d6066423c as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/9151545f96114815b4d9450d6066423c 2024-11-10T19:19:08,083 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/9151545f96114815b4d9450d6066423c, entries=100, sequenceid=351, filesize=22.0 K 2024-11-10T19:19:08,084 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/B/8d53f673af1e4264be49efa70a5cc321 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8d53f673af1e4264be49efa70a5cc321 2024-11-10T19:19:08,086 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8d53f673af1e4264be49efa70a5cc321, entries=100, sequenceid=351, filesize=9.6 K 2024-11-10T19:19:08,087 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/.tmp/C/83663622396a4350a7d66edbb78695f9 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/83663622396a4350a7d66edbb78695f9 2024-11-10T19:19:08,089 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/83663622396a4350a7d66edbb78695f9, entries=100, sequenceid=351, filesize=9.6 K 2024-11-10T19:19:08,089 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4988256ba260015146113518269b8182 in 1642ms, sequenceid=351, compaction requested=true 2024-11-10T19:19:08,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/ac2ec5d8abfa479a8d90b775946bc498, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7dea3822a18d42a683a3b8a4cc655e3a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0e4c5e5257864113875e4a8d94ff1cfd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/483e3fab27bc4ab7814df63833a2d507, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/998e5e5ba4d14569af2639b03b11f7ad, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d473bacada84d09b86a11ce621c387f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/b6d42d99e7a445deb96e289344295116, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0f985502107744f48cf56ffd8e5efa63, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/33344bb584f3486490b83838e4f8837e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1ad8ac29fc024577b53a7bf987bb02ac, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/f0d49873c3a1433dbb86ddda5638106b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0cd251d916c54c30bc45b2b68b7414fd, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/54bfbf17732147f1a15f850e0ce1f338, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d756fb108a640deb1b60408e96a1dd4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1aca7f0f4f484642bc7fa577fb25d40d, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/c85fabb85cc14833bd116e80cc95720c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/737f22cc5eb940f18ad02fd71cf23ac1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/3e185a940e844663a1d4a837407b9992, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/d91dc363228146579e31608b6b2c7b28, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/23723773ad554c778626252ca86ee73c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/91315a1337954d2f89cce9140f55cfae, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/df71a70812c44f5dbdb7f306cd015aaf] to archive 2024-11-10T19:19:08,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:19:08,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/ac2ec5d8abfa479a8d90b775946bc498 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/ac2ec5d8abfa479a8d90b775946bc498 2024-11-10T19:19:08,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7dea3822a18d42a683a3b8a4cc655e3a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7dea3822a18d42a683a3b8a4cc655e3a 2024-11-10T19:19:08,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0e4c5e5257864113875e4a8d94ff1cfd to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0e4c5e5257864113875e4a8d94ff1cfd 2024-11-10T19:19:08,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/483e3fab27bc4ab7814df63833a2d507 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/483e3fab27bc4ab7814df63833a2d507 2024-11-10T19:19:08,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/998e5e5ba4d14569af2639b03b11f7ad to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/998e5e5ba4d14569af2639b03b11f7ad 2024-11-10T19:19:08,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d473bacada84d09b86a11ce621c387f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d473bacada84d09b86a11ce621c387f 2024-11-10T19:19:08,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/b6d42d99e7a445deb96e289344295116 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/b6d42d99e7a445deb96e289344295116 2024-11-10T19:19:08,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0f985502107744f48cf56ffd8e5efa63 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0f985502107744f48cf56ffd8e5efa63 2024-11-10T19:19:08,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/33344bb584f3486490b83838e4f8837e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/33344bb584f3486490b83838e4f8837e 2024-11-10T19:19:08,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1ad8ac29fc024577b53a7bf987bb02ac to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1ad8ac29fc024577b53a7bf987bb02ac 2024-11-10T19:19:08,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/f0d49873c3a1433dbb86ddda5638106b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/f0d49873c3a1433dbb86ddda5638106b 2024-11-10T19:19:08,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0cd251d916c54c30bc45b2b68b7414fd to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0cd251d916c54c30bc45b2b68b7414fd 2024-11-10T19:19:08,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/54bfbf17732147f1a15f850e0ce1f338 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/54bfbf17732147f1a15f850e0ce1f338 2024-11-10T19:19:08,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d756fb108a640deb1b60408e96a1dd4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/7d756fb108a640deb1b60408e96a1dd4 2024-11-10T19:19:08,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1aca7f0f4f484642bc7fa577fb25d40d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/1aca7f0f4f484642bc7fa577fb25d40d 2024-11-10T19:19:08,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/c85fabb85cc14833bd116e80cc95720c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/c85fabb85cc14833bd116e80cc95720c 2024-11-10T19:19:08,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/737f22cc5eb940f18ad02fd71cf23ac1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/737f22cc5eb940f18ad02fd71cf23ac1 2024-11-10T19:19:08,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/3e185a940e844663a1d4a837407b9992 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/3e185a940e844663a1d4a837407b9992 2024-11-10T19:19:08,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/d91dc363228146579e31608b6b2c7b28 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/d91dc363228146579e31608b6b2c7b28 2024-11-10T19:19:08,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/23723773ad554c778626252ca86ee73c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/23723773ad554c778626252ca86ee73c 2024-11-10T19:19:08,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/91315a1337954d2f89cce9140f55cfae to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/91315a1337954d2f89cce9140f55cfae 2024-11-10T19:19:08,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/df71a70812c44f5dbdb7f306cd015aaf to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/df71a70812c44f5dbdb7f306cd015aaf 2024-11-10T19:19:08,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/52bfb3c9ffb34f4d8f63fff1ada55123, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a208690d97c24d13b3d450f8fe5866e9, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/c9adff1f5d704c3498ea1c763540b695, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b9fab8cde9bd474cb50a2f9a8fb87b08, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/60704bba4c9f486c9624960032c71b1a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/06234cdd19504f97b114da8abc86687f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ccd91b97f3814238b2a0cd8dde79a5ad, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/5dbcd9b9396b4653bf4f0fc879d15b93, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/4a6e482b5b714f9b9b7f24e9e6eff7ee, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/99ecbce75f054f639076becb53a584a4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b09a652740db404a9638816a3883ff90, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a883575f9d8c4df29c7da8ee3775c283, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3bc7df88e80849af8ebc83da8fb80967, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/842656c8cdfa4488aaa0691a4f1571fa, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8e3d963b6b7e4c2e8eb3a239f115a61c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ce57059627a74fcb801e245945f8dc8e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/00286edd7fc247919706161a47b3a461, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/61cd3f24584a4776acfc460f4a62eae7, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3621004ee4f3425d9128557a3ba3bd33, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/abfc76495434457f92d6746d567afc62, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/65e5ca2daf4f4a38883d80cc7d22b087, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/6346dd0d3f0547c78a54cd15574c756d] to archive 2024-11-10T19:19:08,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:19:08,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/52bfb3c9ffb34f4d8f63fff1ada55123 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/52bfb3c9ffb34f4d8f63fff1ada55123 2024-11-10T19:19:08,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a208690d97c24d13b3d450f8fe5866e9 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a208690d97c24d13b3d450f8fe5866e9 2024-11-10T19:19:08,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/c9adff1f5d704c3498ea1c763540b695 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/c9adff1f5d704c3498ea1c763540b695 2024-11-10T19:19:08,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b9fab8cde9bd474cb50a2f9a8fb87b08 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b9fab8cde9bd474cb50a2f9a8fb87b08 2024-11-10T19:19:08,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/60704bba4c9f486c9624960032c71b1a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/60704bba4c9f486c9624960032c71b1a 2024-11-10T19:19:08,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/06234cdd19504f97b114da8abc86687f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/06234cdd19504f97b114da8abc86687f 2024-11-10T19:19:08,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ccd91b97f3814238b2a0cd8dde79a5ad to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ccd91b97f3814238b2a0cd8dde79a5ad 2024-11-10T19:19:08,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/5dbcd9b9396b4653bf4f0fc879d15b93 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/5dbcd9b9396b4653bf4f0fc879d15b93 2024-11-10T19:19:08,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/4a6e482b5b714f9b9b7f24e9e6eff7ee to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/4a6e482b5b714f9b9b7f24e9e6eff7ee 2024-11-10T19:19:08,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/99ecbce75f054f639076becb53a584a4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/99ecbce75f054f639076becb53a584a4 2024-11-10T19:19:08,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b09a652740db404a9638816a3883ff90 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/b09a652740db404a9638816a3883ff90 2024-11-10T19:19:08,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a883575f9d8c4df29c7da8ee3775c283 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/a883575f9d8c4df29c7da8ee3775c283 2024-11-10T19:19:08,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3bc7df88e80849af8ebc83da8fb80967 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3bc7df88e80849af8ebc83da8fb80967 2024-11-10T19:19:08,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/842656c8cdfa4488aaa0691a4f1571fa to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/842656c8cdfa4488aaa0691a4f1571fa 2024-11-10T19:19:08,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8e3d963b6b7e4c2e8eb3a239f115a61c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8e3d963b6b7e4c2e8eb3a239f115a61c 2024-11-10T19:19:08,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ce57059627a74fcb801e245945f8dc8e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/ce57059627a74fcb801e245945f8dc8e 2024-11-10T19:19:08,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/00286edd7fc247919706161a47b3a461 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/00286edd7fc247919706161a47b3a461 2024-11-10T19:19:08,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/61cd3f24584a4776acfc460f4a62eae7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/61cd3f24584a4776acfc460f4a62eae7 2024-11-10T19:19:08,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3621004ee4f3425d9128557a3ba3bd33 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/3621004ee4f3425d9128557a3ba3bd33 2024-11-10T19:19:08,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/abfc76495434457f92d6746d567afc62 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/abfc76495434457f92d6746d567afc62 2024-11-10T19:19:08,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/65e5ca2daf4f4a38883d80cc7d22b087 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/65e5ca2daf4f4a38883d80cc7d22b087 2024-11-10T19:19:08,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/6346dd0d3f0547c78a54cd15574c756d to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/6346dd0d3f0547c78a54cd15574c756d 2024-11-10T19:19:08,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/5e97338c23f54eaa9fa817bc4e5d6b4e, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/a6ddb71c05b34b8f9ac444f26a454e50, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/302ef1c1b584486fb296453b06fc65a1, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/06aea2bf80d34802846640a517fa8ed5, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/7158588fd3e443ae8c33a5877cb2a536, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/3082f6699dbe498bbafbd25384e41b5c, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/b6043dfa71c9492ba1cb1fb0af201d4a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/92f1717d90544e9ab39e84950496861f, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/6241837a5f8c4783b1b7515e0877dc2a, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/78db2149187a4ecb88c540907f6634ef, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/760e4270d036481ca9da0b4714bcff82, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/19e5bbdc8e2c4467807764eaa2aa68e4, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/c03c7c9ec65c4761a02726bdc1c9ae76, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/179ca4dc481443ba8e5f20b823f68b08, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/21261ed363e845d8a5c1ea16878dc358, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/4a297515620a427e837f4316cc4e47c2, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/1503335f8a484aeea907cee6335bab38, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/9352627a07434c978ec44cd091f0ab1b, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/24783a3506354b3288261cd4de02a4e0, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/714039d3cb514123a62c6073fab7ed05, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/050a6716562a468e936dff0d7288ef98, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/49f1b09f38004cdca9e29f71f6f586ab] to archive 2024-11-10T19:19:08,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T19:19:08,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/5e97338c23f54eaa9fa817bc4e5d6b4e to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/5e97338c23f54eaa9fa817bc4e5d6b4e 2024-11-10T19:19:08,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/a6ddb71c05b34b8f9ac444f26a454e50 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/a6ddb71c05b34b8f9ac444f26a454e50 2024-11-10T19:19:08,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/302ef1c1b584486fb296453b06fc65a1 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/302ef1c1b584486fb296453b06fc65a1 2024-11-10T19:19:08,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/06aea2bf80d34802846640a517fa8ed5 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/06aea2bf80d34802846640a517fa8ed5 2024-11-10T19:19:08,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/7158588fd3e443ae8c33a5877cb2a536 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/7158588fd3e443ae8c33a5877cb2a536 2024-11-10T19:19:08,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/3082f6699dbe498bbafbd25384e41b5c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/3082f6699dbe498bbafbd25384e41b5c 2024-11-10T19:19:08,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/b6043dfa71c9492ba1cb1fb0af201d4a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/b6043dfa71c9492ba1cb1fb0af201d4a 2024-11-10T19:19:08,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/92f1717d90544e9ab39e84950496861f to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/92f1717d90544e9ab39e84950496861f 2024-11-10T19:19:08,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/6241837a5f8c4783b1b7515e0877dc2a to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/6241837a5f8c4783b1b7515e0877dc2a 2024-11-10T19:19:08,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/78db2149187a4ecb88c540907f6634ef to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/78db2149187a4ecb88c540907f6634ef 2024-11-10T19:19:08,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/760e4270d036481ca9da0b4714bcff82 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/760e4270d036481ca9da0b4714bcff82 2024-11-10T19:19:08,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/19e5bbdc8e2c4467807764eaa2aa68e4 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/19e5bbdc8e2c4467807764eaa2aa68e4 2024-11-10T19:19:08,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/c03c7c9ec65c4761a02726bdc1c9ae76 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/c03c7c9ec65c4761a02726bdc1c9ae76 2024-11-10T19:19:08,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/179ca4dc481443ba8e5f20b823f68b08 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/179ca4dc481443ba8e5f20b823f68b08 2024-11-10T19:19:08,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/21261ed363e845d8a5c1ea16878dc358 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/21261ed363e845d8a5c1ea16878dc358 2024-11-10T19:19:08,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/4a297515620a427e837f4316cc4e47c2 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/4a297515620a427e837f4316cc4e47c2 2024-11-10T19:19:08,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/1503335f8a484aeea907cee6335bab38 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/1503335f8a484aeea907cee6335bab38 2024-11-10T19:19:08,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/9352627a07434c978ec44cd091f0ab1b to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/9352627a07434c978ec44cd091f0ab1b 2024-11-10T19:19:08,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/24783a3506354b3288261cd4de02a4e0 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/24783a3506354b3288261cd4de02a4e0 2024-11-10T19:19:08,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/714039d3cb514123a62c6073fab7ed05 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/714039d3cb514123a62c6073fab7ed05 2024-11-10T19:19:08,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/050a6716562a468e936dff0d7288ef98 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/050a6716562a468e936dff0d7288ef98 2024-11-10T19:19:08,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/49f1b09f38004cdca9e29f71f6f586ab to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/49f1b09f38004cdca9e29f71f6f586ab 2024-11-10T19:19:08,151 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/recovered.edits/354.seqid, newMaxSeqId=354, maxSeqId=4 2024-11-10T19:19:08,151 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. 2024-11-10T19:19:08,151 DEBUG [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1676): Region close journal for 4988256ba260015146113518269b8182: Waiting for close lock at 1731266345723Running coprocessor pre-close hooks at 1731266345723Disabling compacts and flushes for region at 1731266345723Disabling writes for close at 1731266346447 (+724 ms)Obtaining lock to block concurrent updates at 1731266346447Preparing flush snapshotting stores in 4988256ba260015146113518269b8182 at 1731266346447Finished memstore snapshotting TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731266346447Flushing stores of TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182. at 1731266346448 (+1 ms)Flushing 4988256ba260015146113518269b8182/A: creating writer at 1731266346448Flushing mob file 4988256ba260015146113518269b8182/A: appending metadata at 1731266346451 (+3 ms)Flushing mob file 4988256ba260015146113518269b8182/A: closing flushed file at 1731266346452 (+1 ms)Flushing 4988256ba260015146113518269b8182/A: appending metadata at 1731266346858 (+406 ms)Flushing 4988256ba260015146113518269b8182/A: closing flushed file at 1731266346858Flushing 4988256ba260015146113518269b8182/B: creating writer at 1731266347264 (+406 ms)Flushing 4988256ba260015146113518269b8182/B: appending metadata at 1731266347266 (+2 ms)Flushing 4988256ba260015146113518269b8182/B: closing flushed file at 1731266347266Flushing 4988256ba260015146113518269b8182/C: creating writer at 1731266347672 (+406 ms)Flushing 4988256ba260015146113518269b8182/C: appending metadata at 1731266347674 (+2 ms)Flushing 4988256ba260015146113518269b8182/C: closing flushed file at 1731266347674Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c85200: reopening flushed file at 1731266348080 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28151d05: reopening flushed file at 1731266348084 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5959f859: reopening flushed file at 1731266348086 (+2 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4988256ba260015146113518269b8182 in 1642ms, sequenceid=351, compaction requested=true at 1731266348090 (+4 ms)Writing region close event to WAL at 1731266348149 (+59 ms)Running coprocessor post-close hooks at 1731266348151 (+2 ms)Closed at 1731266348151 2024-11-10T19:19:08,153 INFO [RS_CLOSE_REGION-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(157): Closed 4988256ba260015146113518269b8182 2024-11-10T19:19:08,153 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=4988256ba260015146113518269b8182, regionState=CLOSED 2024-11-10T19:19:08,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 because future has completed 2024-11-10T19:19:08,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-10T19:19:08,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseRegionProcedure 4988256ba260015146113518269b8182, server=bdc25462c9ef,34487,1731266167729 in 2.5840 sec 2024-11-10T19:19:08,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-10T19:19:08,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4988256ba260015146113518269b8182, UNASSIGN in 2.5870 sec 2024-11-10T19:19:08,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-10T19:19:08,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5890 sec 2024-11-10T19:19:08,160 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731266348159"}]},"ts":"1731266348159"} 2024-11-10T19:19:08,160 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T19:19:08,160 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T19:19:08,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5980 sec 2024-11-10T19:19:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T19:19:09,731 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:19:09,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T19:19:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:19:09,733 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:19:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T19:19:09,734 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=156, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:19:09,735 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182 2024-11-10T19:19:09,737 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C, FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/recovered.edits] 2024-11-10T19:19:09,738 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0ce7ca797d834f958d6e7d17b2d8d947 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/0ce7ca797d834f958d6e7d17b2d8d947 2024-11-10T19:19:09,739 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/72b5d8f05fb543edb05dc45ce5e680f7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/72b5d8f05fb543edb05dc45ce5e680f7 2024-11-10T19:19:09,740 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/9151545f96114815b4d9450d6066423c to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/A/9151545f96114815b4d9450d6066423c 2024-11-10T19:19:09,742 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8d53f673af1e4264be49efa70a5cc321 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/8d53f673af1e4264be49efa70a5cc321 2024-11-10T19:19:09,742 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/92f74b467c5641d69bb04637abc0c7f8 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/92f74b467c5641d69bb04637abc0c7f8 2024-11-10T19:19:09,743 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/9d51d7276b4b4f20afd539ebe460b0f6 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/B/9d51d7276b4b4f20afd539ebe460b0f6 2024-11-10T19:19:09,745 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/83663622396a4350a7d66edbb78695f9 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/83663622396a4350a7d66edbb78695f9 2024-11-10T19:19:09,745 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/977df2ec9456450eb41ee41a710695ae to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/977df2ec9456450eb41ee41a710695ae 2024-11-10T19:19:09,746 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/d1066498e43a417686caa0d8fe86b9d7 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/C/d1066498e43a417686caa0d8fe86b9d7 2024-11-10T19:19:09,748 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/recovered.edits/354.seqid to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182/recovered.edits/354.seqid 2024-11-10T19:19:09,748 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/default/TestAcidGuarantees/4988256ba260015146113518269b8182 2024-11-10T19:19:09,748 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T19:19:09,749 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T19:19:09,749 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T19:19:09,751 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100c8fc3fdd85449e9af0b04749d98e538_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100c8fc3fdd85449e9af0b04749d98e538_4988256ba260015146113518269b8182 2024-11-10T19:19:09,751 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101b00b82293864f489d88bae82a737575_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101b00b82293864f489d88bae82a737575_4988256ba260015146113518269b8182 2024-11-10T19:19:09,752 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e6a9c85c9c14f5eb89f21e4b8286f3d_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e6a9c85c9c14f5eb89f21e4b8286f3d_4988256ba260015146113518269b8182 2024-11-10T19:19:09,753 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101f05813cc6ee49a9921105978bd48412_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101f05813cc6ee49a9921105978bd48412_4988256ba260015146113518269b8182 2024-11-10T19:19:09,753 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110395b1e6034a9444cb66ef17a37e814c2_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110395b1e6034a9444cb66ef17a37e814c2_4988256ba260015146113518269b8182 2024-11-10T19:19:09,754 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110407be605e49a4d2b8692dadf76f9b5ce_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110407be605e49a4d2b8692dadf76f9b5ce_4988256ba260015146113518269b8182 2024-11-10T19:19:09,754 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110428fbf20f35040eca577c3f785f53439_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110428fbf20f35040eca577c3f785f53439_4988256ba260015146113518269b8182 2024-11-10T19:19:09,755 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111048bc424124cc48d5ae5c9e64a336f9f2_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111048bc424124cc48d5ae5c9e64a336f9f2_4988256ba260015146113518269b8182 2024-11-10T19:19:09,755 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111050e1c6163b714fb38ed3654ed361c6d9_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111050e1c6163b714fb38ed3654ed361c6d9_4988256ba260015146113518269b8182 2024-11-10T19:19:09,756 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111051f777d449cb477bb6f908217452b09e_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111051f777d449cb477bb6f908217452b09e_4988256ba260015146113518269b8182 2024-11-10T19:19:09,757 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111058c91e8df21244a0b08bd0e0c708bc67_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111058c91e8df21244a0b08bd0e0c708bc67_4988256ba260015146113518269b8182 2024-11-10T19:19:09,757 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110668d732c3f414e7faa42de31021b2f14_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110668d732c3f414e7faa42de31021b2f14_4988256ba260015146113518269b8182 2024-11-10T19:19:09,758 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111078471b70dae74c458ce4836b8a907892_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111078471b70dae74c458ce4836b8a907892_4988256ba260015146113518269b8182 2024-11-10T19:19:09,759 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108434d7ce618b48e6914751e5d5670b16_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108434d7ce618b48e6914751e5d5670b16_4988256ba260015146113518269b8182 2024-11-10T19:19:09,759 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111099c9e191e2f2457d89a966f687f068d8_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111099c9e191e2f2457d89a966f687f068d8_4988256ba260015146113518269b8182 2024-11-10T19:19:09,760 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a508e36356274f64b1eed6b7c1b0bf00_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a508e36356274f64b1eed6b7c1b0bf00_4988256ba260015146113518269b8182 2024-11-10T19:19:09,761 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bd20113039164d40ae130d13ec5df771_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bd20113039164d40ae130d13ec5df771_4988256ba260015146113518269b8182 2024-11-10T19:19:09,761 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d7d9fd5b338240fd89106e35bd6d7307_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d7d9fd5b338240fd89106e35bd6d7307_4988256ba260015146113518269b8182 2024-11-10T19:19:09,762 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dcb6e43aae3847979ab41ae746240aa4_4988256ba260015146113518269b8182 to hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dcb6e43aae3847979ab41ae746240aa4_4988256ba260015146113518269b8182 2024-11-10T19:19:09,762 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T19:19:09,764 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=156, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:19:09,766 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T19:19:09,767 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T19:19:09,768 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=156, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:19:09,768 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T19:19:09,769 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731266349768"}]},"ts":"9223372036854775807"} 2024-11-10T19:19:09,770 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T19:19:09,770 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4988256ba260015146113518269b8182, NAME => 'TestAcidGuarantees,,1731266322575.4988256ba260015146113518269b8182.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T19:19:09,770 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T19:19:09,770 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731266349770"}]},"ts":"9223372036854775807"} 2024-11-10T19:19:09,771 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T19:19:09,772 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=156, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T19:19:09,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-10T19:19:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T19:19:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T19:19:09,841 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T19:19:09,850 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249 (was 250), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=692 (was 735), ProcessCount=11 (was 11), AvailableMemoryMB=1103 (was 1413) 2024-11-10T19:19:09,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T19:19:09,850 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:19:09,850 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:19:09,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:09,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:09,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:19:09,850 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T19:19:09,850 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1570076000, stopped=false 2024-11-10T19:19:09,851 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bdc25462c9ef,46761,1731266166843 2024-11-10T19:19:09,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T19:19:09,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T19:19:09,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:19:09,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:19:09,864 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T19:19:09,864 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:19:09,864 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:19:09,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:19:09,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:09,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:19:09,865 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bdc25462c9ef,34487,1731266167729' ***** 2024-11-10T19:19:09,865 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T19:19:09,865 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T19:19:09,865 INFO [RS:0;bdc25462c9ef:34487 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T19:19:09,865 INFO [RS:0;bdc25462c9ef:34487 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T19:19:09,865 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T19:19:09,865 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(959): stopping server bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:09,865 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T19:19:09,866 INFO [RS:0;bdc25462c9ef:34487 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bdc25462c9ef:34487. 2024-11-10T19:19:09,866 DEBUG [RS:0;bdc25462c9ef:34487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:19:09,866 DEBUG [RS:0;bdc25462c9ef:34487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:09,866 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T19:19:09,866 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T19:19:09,866 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T19:19:09,866 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T19:19:09,866 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T19:19:09,866 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T19:19:09,867 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T19:19:09,867 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T19:19:09,867 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T19:19:09,867 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T19:19:09,867 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T19:19:09,867 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T19:19:09,867 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T19:19:09,890 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/info/0034d065e8e54b01b9e68e53171d2e15 is 95, key is TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5./info:/1731266202158/DeleteFamily/seqid=0 2024-11-10T19:19:09,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742486_1662 (size=6514) 2024-11-10T19:19:09,906 INFO [regionserver/bdc25462c9ef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:19:10,067 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T19:19:10,267 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T19:19:10,293 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/info/0034d065e8e54b01b9e68e53171d2e15 2024-11-10T19:19:10,315 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/ns/ef6b1453bd904d1189081f3a0655beb4 is 93, key is TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5./ns:/1731266202088/DeleteFamily/seqid=0 2024-11-10T19:19:10,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742487_1663 (size=5906) 2024-11-10T19:19:10,467 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T19:19:10,667 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T19:19:10,719 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/ns/ef6b1453bd904d1189081f3a0655beb4 2024-11-10T19:19:10,738 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/rep_barrier/cce8cc41a0064109b99d016690327372 is 102, key is TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5./rep_barrier:/1731266202088/DeleteFamily/seqid=0 2024-11-10T19:19:10,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742488_1664 (size=6025) 2024-11-10T19:19:10,851 INFO [regionserver/bdc25462c9ef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T19:19:10,868 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T19:19:10,876 INFO [regionserver/bdc25462c9ef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T19:19:10,876 INFO [regionserver/bdc25462c9ef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-10T19:19:11,068 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T19:19:11,142 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/rep_barrier/cce8cc41a0064109b99d016690327372 2024-11-10T19:19:11,158 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/table/c1e189451fd7422d9d948543c020a8c7 is 96, key is TestAcidGuarantees,,1731266172539.4f5e56bdcde85bbc88e3b9e5355a12b5./table:/1731266202088/DeleteFamily/seqid=0 2024-11-10T19:19:11,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742489_1665 (size=5936) 2024-11-10T19:19:11,268 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T19:19:11,468 DEBUG [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T19:19:11,562 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/table/c1e189451fd7422d9d948543c020a8c7 2024-11-10T19:19:11,565 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/info/0034d065e8e54b01b9e68e53171d2e15 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/info/0034d065e8e54b01b9e68e53171d2e15 2024-11-10T19:19:11,567 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/info/0034d065e8e54b01b9e68e53171d2e15, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T19:19:11,568 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/ns/ef6b1453bd904d1189081f3a0655beb4 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/ns/ef6b1453bd904d1189081f3a0655beb4 2024-11-10T19:19:11,570 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/ns/ef6b1453bd904d1189081f3a0655beb4, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T19:19:11,571 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/rep_barrier/cce8cc41a0064109b99d016690327372 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/rep_barrier/cce8cc41a0064109b99d016690327372 2024-11-10T19:19:11,573 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/rep_barrier/cce8cc41a0064109b99d016690327372, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T19:19:11,574 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/.tmp/table/c1e189451fd7422d9d948543c020a8c7 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/table/c1e189451fd7422d9d948543c020a8c7 2024-11-10T19:19:11,576 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/table/c1e189451fd7422d9d948543c020a8c7, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T19:19:11,577 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false 2024-11-10T19:19:11,580 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T19:19:11,580 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T19:19:11,580 INFO [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T19:19:11,580 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731266349866Running coprocessor pre-close hooks at 1731266349867 (+1 ms)Disabling compacts and flushes for region at 1731266349867Disabling writes for close at 1731266349867Obtaining lock to block concurrent updates at 1731266349867Preparing flush snapshotting stores in 1588230740 at 1731266349867Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731266349867Flushing stores of hbase:meta,,1.1588230740 at 1731266349868 (+1 ms)Flushing 1588230740/info: creating writer at 1731266349868Flushing 1588230740/info: appending metadata at 1731266349889 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731266349889Flushing 1588230740/ns: creating writer at 1731266350297 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731266350315 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731266350315Flushing 1588230740/rep_barrier: creating writer at 1731266350721 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731266350738 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731266350738Flushing 1588230740/table: creating writer at 1731266351145 (+407 ms)Flushing 1588230740/table: appending metadata at 1731266351158 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731266351158Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22c550a0: reopening flushed file at 1731266351564 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36bfe903: reopening flushed file at 1731266351568 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cc8e917: reopening flushed file at 1731266351570 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@539e35cb: reopening flushed file at 1731266351573 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false at 1731266351577 (+4 ms)Writing region close event to WAL at 1731266351578 (+1 ms)Running coprocessor post-close hooks at 1731266351580 (+2 ms)Closed at 1731266351580 2024-11-10T19:19:11,580 DEBUG [RS_CLOSE_META-regionserver/bdc25462c9ef:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T19:19:11,668 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(976): stopping server bdc25462c9ef,34487,1731266167729; all regions closed. 2024-11-10T19:19:11,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741834_1010 (size=25344) 2024-11-10T19:19:11,674 DEBUG [RS:0;bdc25462c9ef:34487 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/oldWALs 2024-11-10T19:19:11,674 INFO [RS:0;bdc25462c9ef:34487 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bdc25462c9ef%2C34487%2C1731266167729.meta:.meta(num 1731266172097) 2024-11-10T19:19:11,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741832_1008 (size=18262211) 2024-11-10T19:19:11,677 DEBUG [RS:0;bdc25462c9ef:34487 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/oldWALs 2024-11-10T19:19:11,677 INFO [RS:0;bdc25462c9ef:34487 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bdc25462c9ef%2C34487%2C1731266167729:(num 1731266171021) 2024-11-10T19:19:11,677 DEBUG [RS:0;bdc25462c9ef:34487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:19:11,677 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:19:11,677 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T19:19:11,677 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.ChoreService(370): Chore service for: regionserver/bdc25462c9ef:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T19:19:11,677 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T19:19:11,678 INFO [regionserver/bdc25462c9ef:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T19:19:11,678 INFO [RS:0;bdc25462c9ef:34487 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34487 2024-11-10T19:19:11,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:19:11,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bdc25462c9ef,34487,1731266167729 2024-11-10T19:19:11,685 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T19:19:11,685 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bdc25462c9ef,34487,1731266167729] 2024-11-10T19:19:11,695 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bdc25462c9ef,34487,1731266167729 already deleted, retry=false 2024-11-10T19:19:11,695 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bdc25462c9ef,34487,1731266167729 expired; onlineServers=0 2024-11-10T19:19:11,695 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bdc25462c9ef,46761,1731266166843' ***** 2024-11-10T19:19:11,695 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T19:19:11,695 INFO [M:0;bdc25462c9ef:46761 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T19:19:11,695 INFO [M:0;bdc25462c9ef:46761 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T19:19:11,695 DEBUG [M:0;bdc25462c9ef:46761 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T19:19:11,696 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T19:19:11,696 DEBUG [M:0;bdc25462c9ef:46761 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T19:19:11,696 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster-HFileCleaner.large.0-1731266170664 {}] cleaner.HFileCleaner(306): Exit Thread[master/bdc25462c9ef:0:becomeActiveMaster-HFileCleaner.large.0-1731266170664,5,FailOnTimeoutGroup] 2024-11-10T19:19:11,696 DEBUG [master/bdc25462c9ef:0:becomeActiveMaster-HFileCleaner.small.0-1731266170665 {}] cleaner.HFileCleaner(306): Exit Thread[master/bdc25462c9ef:0:becomeActiveMaster-HFileCleaner.small.0-1731266170665,5,FailOnTimeoutGroup] 2024-11-10T19:19:11,696 INFO [M:0;bdc25462c9ef:46761 {}] hbase.ChoreService(370): Chore service for: master/bdc25462c9ef:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T19:19:11,696 INFO [M:0;bdc25462c9ef:46761 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T19:19:11,696 DEBUG [M:0;bdc25462c9ef:46761 {}] master.HMaster(1795): Stopping service threads 2024-11-10T19:19:11,696 INFO [M:0;bdc25462c9ef:46761 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T19:19:11,696 INFO [M:0;bdc25462c9ef:46761 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T19:19:11,696 ERROR [M:0;bdc25462c9ef:46761 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T19:19:11,697 INFO [M:0;bdc25462c9ef:46761 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T19:19:11,697 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T19:19:11,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T19:19:11,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:19:11,706 DEBUG [M:0;bdc25462c9ef:46761 {}] zookeeper.ZKUtil(347): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T19:19:11,706 WARN [M:0;bdc25462c9ef:46761 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T19:19:11,706 INFO [M:0;bdc25462c9ef:46761 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/.lastflushedseqids 2024-11-10T19:19:11,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742490_1666 (size=72) 2024-11-10T19:19:11,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:19:11,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34487-0x101262d38720001, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:19:11,796 INFO [RS:0;bdc25462c9ef:34487 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T19:19:11,796 INFO [RS:0;bdc25462c9ef:34487 {}] regionserver.HRegionServer(1031): Exiting; stopping=bdc25462c9ef,34487,1731266167729; zookeeper connection closed. 2024-11-10T19:19:11,796 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6fc807fd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6fc807fd 2024-11-10T19:19:11,797 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T19:19:12,116 INFO [M:0;bdc25462c9ef:46761 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T19:19:12,116 INFO [M:0;bdc25462c9ef:46761 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T19:19:12,116 DEBUG [M:0;bdc25462c9ef:46761 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T19:19:12,116 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:19:12,116 DEBUG [M:0;bdc25462c9ef:46761 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:19:12,116 DEBUG [M:0;bdc25462c9ef:46761 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T19:19:12,116 DEBUG [M:0;bdc25462c9ef:46761 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:19:12,116 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=766.31 KB heapSize=937.73 KB 2024-11-10T19:19:12,135 DEBUG [M:0;bdc25462c9ef:46761 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bf4b62f24fb4ce0b59b43d493a77dee is 82, key is hbase:meta,,1/info:regioninfo/1731266172188/Put/seqid=0 2024-11-10T19:19:12,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742491_1667 (size=5672) 2024-11-10T19:19:12,539 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2125 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bf4b62f24fb4ce0b59b43d493a77dee 2024-11-10T19:19:12,558 DEBUG [M:0;bdc25462c9ef:46761 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/278952b68056448c98d9c41be285f65d is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x19/proc:d/1731266203982/Put/seqid=0 2024-11-10T19:19:12,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742492_1668 (size=42930) 2024-11-10T19:19:12,962 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=765.76 KB at sequenceid=2125 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/278952b68056448c98d9c41be285f65d 2024-11-10T19:19:12,965 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 278952b68056448c98d9c41be285f65d 2024-11-10T19:19:12,982 DEBUG [M:0;bdc25462c9ef:46761 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf912268df7e48a78984471da3a644c0 is 69, key is bdc25462c9ef,34487,1731266167729/rs:state/1731266170745/Put/seqid=0 2024-11-10T19:19:12,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073742493_1669 (size=5156) 2024-11-10T19:19:13,385 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2125 (bloomFilter=true), to=hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf912268df7e48a78984471da3a644c0 2024-11-10T19:19:13,388 DEBUG [M:0;bdc25462c9ef:46761 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bf4b62f24fb4ce0b59b43d493a77dee as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1bf4b62f24fb4ce0b59b43d493a77dee 2024-11-10T19:19:13,391 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1bf4b62f24fb4ce0b59b43d493a77dee, entries=8, sequenceid=2125, filesize=5.5 K 2024-11-10T19:19:13,392 DEBUG [M:0;bdc25462c9ef:46761 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/278952b68056448c98d9c41be285f65d as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/278952b68056448c98d9c41be285f65d 2024-11-10T19:19:13,394 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 278952b68056448c98d9c41be285f65d 2024-11-10T19:19:13,394 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/278952b68056448c98d9c41be285f65d, entries=156, sequenceid=2125, filesize=41.9 K 2024-11-10T19:19:13,394 DEBUG [M:0;bdc25462c9ef:46761 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf912268df7e48a78984471da3a644c0 as hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bf912268df7e48a78984471da3a644c0 2024-11-10T19:19:13,397 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34609/user/jenkins/test-data/eca217ff-ca46-aa20-8dc6-bf2634483af0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bf912268df7e48a78984471da3a644c0, entries=1, sequenceid=2125, filesize=5.0 K 2024-11-10T19:19:13,397 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.HRegion(3140): Finished flush of dataSize ~766.31 KB/784704, heapSize ~937.44 KB/959936, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1281ms, sequenceid=2125, compaction requested=false 2024-11-10T19:19:13,399 INFO [M:0;bdc25462c9ef:46761 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:19:13,399 DEBUG [M:0;bdc25462c9ef:46761 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731266352116Disabling compacts and flushes for region at 1731266352116Disabling writes for close at 1731266352116Obtaining lock to block concurrent updates at 1731266352116Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731266352116Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=784704, getHeapSize=960176, getOffHeapSize=0, getCellsCount=2306 at 1731266352117 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731266352117Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731266352117Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731266352135 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731266352135Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731266352542 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731266352558 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731266352558Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731266352965 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731266352981 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731266352981Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57cea4bb: reopening flushed file at 1731266353388 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@652b0489: reopening flushed file at 1731266353391 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@174b1cce: reopening flushed file at 1731266353394 (+3 ms)Finished flush of dataSize ~766.31 KB/784704, heapSize ~937.44 KB/959936, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1281ms, sequenceid=2125, compaction requested=false at 1731266353397 (+3 ms)Writing region close event to WAL at 1731266353399 (+2 ms)Closed at 1731266353399 2024-11-10T19:19:13,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38057 is added to blk_1073741830_1006 (size=923972) 2024-11-10T19:19:13,401 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T19:19:13,401 INFO [M:0;bdc25462c9ef:46761 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T19:19:13,401 INFO [M:0;bdc25462c9ef:46761 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46761 2024-11-10T19:19:13,402 INFO [M:0;bdc25462c9ef:46761 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T19:19:13,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:19:13,511 INFO [M:0;bdc25462c9ef:46761 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T19:19:13,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46761-0x101262d38720000, quorum=127.0.0.1:53441, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:19:13,518 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T19:19:13,520 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T19:19:13,521 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T19:19:13,521 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T19:19:13,521 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/hadoop.log.dir/,STOPPED} 2024-11-10T19:19:13,523 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T19:19:13,523 WARN [BP-1526193424-172.17.0.3-1731266162412 heartbeating to localhost/127.0.0.1:34609 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T19:19:13,523 WARN [BP-1526193424-172.17.0.3-1731266162412 heartbeating to localhost/127.0.0.1:34609 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1526193424-172.17.0.3-1731266162412 (Datanode Uuid fb2a8015-4173-44ad-935d-47d25a540c3c) service to localhost/127.0.0.1:34609 2024-11-10T19:19:13,523 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T19:19:13,526 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41/data/data1/current/BP-1526193424-172.17.0.3-1731266162412 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T19:19:13,526 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/cluster_13af8a00-4f99-265d-0dcb-d48a52405a41/data/data2/current/BP-1526193424-172.17.0.3-1731266162412 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T19:19:13,527 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T19:19:13,533 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T19:19:13,534 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T19:19:13,534 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T19:19:13,534 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T19:19:13,534 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/893bc08e-2bd6-0bc2-873e-745b4e439aba/hadoop.log.dir/,STOPPED} 2024-11-10T19:19:13,554 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T19:19:13,665 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T19:19:13,667 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T19:19:13,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down